Commit ·
2a49d9a
0
Parent(s):
Duplicate from GGUFGuy/hyperdex-trainer
Browse files- .gitattributes +35 -0
- .gitignore +4 -0
- Dockerfile +29 -0
- README.md +111 -0
- main.py +643 -0
- nanodex/__init__.py +0 -0
- nanodex/config.py +208 -0
- nanodex/data.py +282 -0
- nanodex/db.py +370 -0
- nanodex/estimate.py +118 -0
- nanodex/hub.py +111 -0
- nanodex/store.py +259 -0
- nanodex/trainer.py +254 -0
- nanodex/worker.py +137 -0
- requirements.txt +15 -0
- tokenizer/special_tokens_map.json +5 -0
- tokenizer/tokenizer.json +0 -0
- tokenizer/tokenizer_config.json +27 -0
- web/static/app.css +351 -0
- web/static/app.js +164 -0
- web/templates/about.html +170 -0
- web/templates/base.html +66 -0
- web/templates/detail.html +319 -0
- web/templates/error.html +11 -0
- web/templates/explore.html +147 -0
- web/templates/home.html +71 -0
- web/templates/landing.html +128 -0
- web/templates/models.html +75 -0
- web/templates/profile.html +74 -0
- web/templates/queue.html +120 -0
- web/templates/train.html +273 -0
- web/templates/user.html +100 -0
.gitattributes
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
*.7z filter=lfs diff=lfs merge=lfs -text
|
| 2 |
+
*.arrow filter=lfs diff=lfs merge=lfs -text
|
| 3 |
+
*.bin filter=lfs diff=lfs merge=lfs -text
|
| 4 |
+
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
| 5 |
+
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
| 6 |
+
*.ftz filter=lfs diff=lfs merge=lfs -text
|
| 7 |
+
*.gz filter=lfs diff=lfs merge=lfs -text
|
| 8 |
+
*.h5 filter=lfs diff=lfs merge=lfs -text
|
| 9 |
+
*.joblib filter=lfs diff=lfs merge=lfs -text
|
| 10 |
+
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
| 11 |
+
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
| 12 |
+
*.model filter=lfs diff=lfs merge=lfs -text
|
| 13 |
+
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
| 14 |
+
*.npy filter=lfs diff=lfs merge=lfs -text
|
| 15 |
+
*.npz filter=lfs diff=lfs merge=lfs -text
|
| 16 |
+
*.onnx filter=lfs diff=lfs merge=lfs -text
|
| 17 |
+
*.ot filter=lfs diff=lfs merge=lfs -text
|
| 18 |
+
*.parquet filter=lfs diff=lfs merge=lfs -text
|
| 19 |
+
*.pb filter=lfs diff=lfs merge=lfs -text
|
| 20 |
+
*.pickle filter=lfs diff=lfs merge=lfs -text
|
| 21 |
+
*.pkl filter=lfs diff=lfs merge=lfs -text
|
| 22 |
+
*.pt filter=lfs diff=lfs merge=lfs -text
|
| 23 |
+
*.pth filter=lfs diff=lfs merge=lfs -text
|
| 24 |
+
*.rar filter=lfs diff=lfs merge=lfs -text
|
| 25 |
+
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
| 26 |
+
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
| 27 |
+
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
| 28 |
+
*.tar filter=lfs diff=lfs merge=lfs -text
|
| 29 |
+
*.tflite filter=lfs diff=lfs merge=lfs -text
|
| 30 |
+
*.tgz filter=lfs diff=lfs merge=lfs -text
|
| 31 |
+
*.wasm filter=lfs diff=lfs merge=lfs -text
|
| 32 |
+
*.xz filter=lfs diff=lfs merge=lfs -text
|
| 33 |
+
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
+
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
+
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
.gitignore
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
runtime/
|
| 2 |
+
__pycache__/
|
| 3 |
+
*.pyc
|
| 4 |
+
.venv/
|
Dockerfile
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
FROM python:3.12-slim
|
| 2 |
+
|
| 3 |
+
ENV PYTHONUNBUFFERED=1 \
|
| 4 |
+
PYTHONDONTWRITEBYTECODE=1 \
|
| 5 |
+
PIP_NO_CACHE_DIR=1 \
|
| 6 |
+
HF_HOME=/home/user/.cache/huggingface \
|
| 7 |
+
HF_HUB_ENABLE_HF_TRANSFER=0 \
|
| 8 |
+
PORT=7860
|
| 9 |
+
|
| 10 |
+
RUN apt-get update && apt-get install -y --no-install-recommends \
|
| 11 |
+
build-essential git curl && \
|
| 12 |
+
rm -rf /var/lib/apt/lists/*
|
| 13 |
+
|
| 14 |
+
RUN useradd -m -u 1000 user
|
| 15 |
+
WORKDIR /home/user/app
|
| 16 |
+
|
| 17 |
+
COPY --chown=user requirements.txt .
|
| 18 |
+
RUN pip install --upgrade pip && pip install -r requirements.txt
|
| 19 |
+
|
| 20 |
+
COPY --chown=user . .
|
| 21 |
+
|
| 22 |
+
RUN mkdir -p /home/user/app/runtime /home/user/.cache/huggingface && \
|
| 23 |
+
chown -R user:user /home/user
|
| 24 |
+
|
| 25 |
+
USER user
|
| 26 |
+
EXPOSE 7860
|
| 27 |
+
|
| 28 |
+
CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "7860", \
|
| 29 |
+
"--workers", "1", "--timeout-keep-alive", "75"]
|
README.md
ADDED
|
@@ -0,0 +1,111 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
title: NanoDex
|
| 3 |
+
emoji: 🧬
|
| 4 |
+
colorFrom: green
|
| 5 |
+
colorTo: blue
|
| 6 |
+
sdk: docker
|
| 7 |
+
app_port: 7860
|
| 8 |
+
pinned: true
|
| 9 |
+
license: apache-2.0
|
| 10 |
+
short_description: Train a decoder-only LM from scratch on fineweb-edu
|
| 11 |
+
hf_oauth: true
|
| 12 |
+
hf_oauth_scopes:
|
| 13 |
+
- read-repos
|
| 14 |
+
- write-repos
|
| 15 |
+
- manage-repos
|
| 16 |
+
hf_oauth_expiration_minutes: 480
|
| 17 |
+
startup_duration_timeout: 45m
|
| 18 |
+
datasets:
|
| 19 |
+
- HuggingFaceFW/fineweb-edu
|
| 20 |
+
---
|
| 21 |
+
|
| 22 |
+
# 🧬 NanoDex
|
| 23 |
+
|
| 24 |
+
**Train your first decoder-only language model from scratch, in the cloud.**
|
| 25 |
+
|
| 26 |
+
Not a fine-tune. Not an adapter. Random weights, a real corpus, and a real loss
|
| 27 |
+
curve you watch fall in real time.
|
| 28 |
+
|
| 29 |
+
Pick a size (**492k / 1.06M / 5.17M / 8.06M** parameters), pick a token budget
|
| 30 |
+
(**200M – 1.5B** tokens of fineweb-edu), and queue it. A worker picks it up, you
|
| 31 |
+
watch the loss fall live, and when it's done the model is pushed to *your*
|
| 32 |
+
Hugging Face account with a generated model card.
|
| 33 |
+
|
| 34 |
+
## Pages
|
| 35 |
+
|
| 36 |
+
| Route | What it is |
|
| 37 |
+
|---|---|
|
| 38 |
+
| `/` | Landing page when signed out; your dashboard when signed in |
|
| 39 |
+
| `/train` | Four-step run wizard — size, tokens, name, review |
|
| 40 |
+
| `/models` | Everything you've trained: training, queued (with position), ready |
|
| 41 |
+
| `/models/{id}` | One run in detail — live loss chart, metrics, logs, publish, playground |
|
| 42 |
+
| `/queue` | Global queue and worker status |
|
| 43 |
+
| `/profile` | Account, totals, permissions |
|
| 44 |
+
| `/about` | The full recipe |
|
| 45 |
+
|
| 46 |
+
## Architecture
|
| 47 |
+
|
| 48 |
+
A standard modern decoder-only transformer (`LlamaForCausalLM`): SiLU MLP,
|
| 49 |
+
RMSNorm (ε=1e-5), rotary position embeddings (θ=10000), grouped-query attention,
|
| 50 |
+
tied input/output embeddings, no biases. The four tiers are that same recipe
|
| 51 |
+
scaled down in width and depth.
|
| 52 |
+
|
| 53 |
+
| Tier | Parameters | Layers | Hidden | Heads (KV) | FFN | Tokens / step |
|
| 54 |
+
|---|---:|---:|---:|---|---:|---:|
|
| 55 |
+
| NanoDex-500k | 492,192 | 3 | 96 | 6 (2) | 256 | 131,072 |
|
| 56 |
+
| NanoDex-1M | 1,062,272 | 5 | 128 | 8 (4) | 288 | 262,144 |
|
| 57 |
+
| NanoDex-5M | 5,172,384 | 9 | 224 | 8 (2) | 592 | 393,216 |
|
| 58 |
+
| NanoDex-8M | 8,060,256 | 9 | 288 | 9 (3) | 704 | 524,288 |
|
| 59 |
+
|
| 60 |
+
A typical small-LM vocabulary (~49k tokens) would be 28M embedding parameters on
|
| 61 |
+
its own — more than three times the biggest model here. So NanoDex ships its own
|
| 62 |
+
**2,048-token byte-level BPE** trained on fineweb-edu (~2.7 chars/token). The
|
| 63 |
+
parameter counts above are real totals, embeddings included.
|
| 64 |
+
|
| 65 |
+
## Data
|
| 66 |
+
|
| 67 |
+
[`HuggingFaceFW/fineweb-edu`](https://huggingface.co/datasets/HuggingFaceFW/fineweb-edu),
|
| 68 |
+
config `sample-10BT`. A 1.505B-token slice is tokenized **once** into a flat
|
| 69 |
+
`uint16` file (~2.8 GB) that every run reads from, so workers are never
|
| 70 |
+
data-bound. Each run starts at its own random offset. Building the cache is a
|
| 71 |
+
one-time cost per container; a run asking for more tokens than are cached yet
|
| 72 |
+
waits for the writer and reports it in the run log.
|
| 73 |
+
|
| 74 |
+
## Training
|
| 75 |
+
|
| 76 |
+
AdamW (β=0.9/0.95, wd=0.1, grad-clip 1.0) · 2% warmup then cosine decay to 10%
|
| 77 |
+
of peak · bf16 autocast · 512-token context · 128–256 sequences per forward
|
| 78 |
+
pass · 131k–524k tokens per optimizer step, with automatic micro-batch backoff
|
| 79 |
+
if CUDA runs out of memory. One worker thread per GPU claims the oldest queued run
|
| 80 |
+
atomically, so a 4-GPU Space trains four models at once.
|
| 81 |
+
|
| 82 |
+
## Layout
|
| 83 |
+
|
| 84 |
+
```
|
| 85 |
+
main.py FastAPI app — pages, JSON API, Hugging Face OAuth
|
| 86 |
+
Dockerfile
|
| 87 |
+
web/templates/ landing, home, train wizard, models, detail, queue,
|
| 88 |
+
profile, about, error
|
| 89 |
+
web/static/ app.css, app.js (dependency-free charts + polling)
|
| 90 |
+
nanodex/config.py tier definitions + exact parameter counting
|
| 91 |
+
nanodex/data.py shared fineweb-edu token cache
|
| 92 |
+
nanodex/trainer.py the training loop
|
| 93 |
+
nanodex/worker.py one worker per GPU
|
| 94 |
+
nanodex/hub.py push a finished run to the user's account
|
| 95 |
+
nanodex/estimate.py ETA, self-calibrating from completed runs
|
| 96 |
+
nanodex/db.py SQLite job store
|
| 97 |
+
tokenizer/ the 2,048-token BPE
|
| 98 |
+
```
|
| 99 |
+
|
| 100 |
+
## A note on scale
|
| 101 |
+
|
| 102 |
+
1.5 billion tokens into an 8M-parameter model is ~9× past Chinchilla-optimal,
|
| 103 |
+
and into a 500k-parameter model over 150× past it — on purpose. Tiny models keep
|
| 104 |
+
improving long past the compute-optimal point, and the goal here isn't FLOP
|
| 105 |
+
efficiency — it's watching cross-entropy fall from 7.6
|
| 106 |
+
(uniform over 2,048 tokens) toward 4, and seeing a network that started as pure
|
| 107 |
+
noise begin to emit English words.
|
| 108 |
+
|
| 109 |
+
Nothing trained here is a useful assistant. That was never the point.
|
| 110 |
+
|
| 111 |
+
Built with 🤗 by [hugging-science](https://huggingface.co/hugging-science).
|
main.py
ADDED
|
@@ -0,0 +1,643 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
NanoDex — a place to pre-train small decoder-only language models from scratch.
|
| 3 |
+
|
| 4 |
+
FastAPI multi-page app:
|
| 5 |
+
/ landing (logged out) / dashboard (logged in)
|
| 6 |
+
/train multi-step run wizard
|
| 7 |
+
/models your runs: training, queued, finished
|
| 8 |
+
/models/{id} one run in detail — live loss, logs, publish, playground
|
| 9 |
+
/profile account + stats
|
| 10 |
+
/login /logout Hugging Face OAuth
|
| 11 |
+
/api/* JSON for live polling
|
| 12 |
+
"""
|
| 13 |
+
|
| 14 |
+
import os
|
| 15 |
+
import secrets
|
| 16 |
+
import sys
|
| 17 |
+
import threading
|
| 18 |
+
import time
|
| 19 |
+
from typing import Optional
|
| 20 |
+
|
| 21 |
+
# Printed before anything heavy so a stalled boot is always diagnosable from
|
| 22 |
+
# the Space's run log.
|
| 23 |
+
print("[boot] NanoDex starting", flush=True)
|
| 24 |
+
|
| 25 |
+
from fastapi import FastAPI, Request, HTTPException, Body
|
| 26 |
+
from fastapi.responses import HTMLResponse, RedirectResponse, JSONResponse
|
| 27 |
+
from fastapi.staticfiles import StaticFiles
|
| 28 |
+
from fastapi.templating import Jinja2Templates
|
| 29 |
+
from starlette.middleware.sessions import SessionMiddleware
|
| 30 |
+
|
| 31 |
+
from nanodex import db, worker, hub, estimate, store
|
| 32 |
+
from nanodex import data as ndata
|
| 33 |
+
from nanodex.config import (TIERS, TIER_ORDER, MIN_TOKENS, MAX_TOKENS,
|
| 34 |
+
TOKEN_STOPS_M, SEQ_LEN, VOCAB_SIZE)
|
| 35 |
+
from nanodex.trainer import total_steps_for
|
| 36 |
+
|
| 37 |
+
BASE = os.path.dirname(os.path.abspath(__file__))
|
| 38 |
+
TOKENIZER_DIR = os.path.join(BASE, "tokenizer")
|
| 39 |
+
MAX_ACTIVE_PER_USER = int(os.environ.get("MAX_ACTIVE_PER_USER", "2"))
|
| 40 |
+
|
| 41 |
+
OAUTH_CLIENT_ID = os.environ.get("OAUTH_CLIENT_ID")
|
| 42 |
+
OAUTH_CLIENT_SECRET = os.environ.get("OAUTH_CLIENT_SECRET")
|
| 43 |
+
OAUTH_SCOPES = os.environ.get("OAUTH_SCOPES", "openid profile read-repos write-repos manage-repos")
|
| 44 |
+
OPENID_PROVIDER_URL = os.environ.get("OPENID_PROVIDER_URL", "https://huggingface.co")
|
| 45 |
+
SPACE_HOST = os.environ.get("SPACE_HOST")
|
| 46 |
+
OAUTH_READY = bool(OAUTH_CLIENT_ID and OAUTH_CLIENT_SECRET)
|
| 47 |
+
|
| 48 |
+
print(f"[boot] data dir: {db.DATA_DIR}", flush=True)
|
| 49 |
+
db.init()
|
| 50 |
+
print("[boot] database ready", flush=True)
|
| 51 |
+
|
| 52 |
+
from tokenizers import Tokenizer as _FastTok
|
| 53 |
+
TOKENIZER = _FastTok.from_file(os.path.join(TOKENIZER_DIR, "tokenizer.json"))
|
| 54 |
+
print(f"[boot] tokenizer loaded ({TOKENIZER.get_vocab_size()} tokens)", flush=True)
|
| 55 |
+
|
| 56 |
+
DEVICES = worker.devices()
|
| 57 |
+
print(f"[boot] devices: {DEVICES}", flush=True)
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
def _restore_history():
|
| 61 |
+
"""
|
| 62 |
+
Pull previously finished runs back out of the bucket. Done in a thread so
|
| 63 |
+
a slow or unreachable bucket never delays the app coming up.
|
| 64 |
+
"""
|
| 65 |
+
try:
|
| 66 |
+
if not store.available():
|
| 67 |
+
return
|
| 68 |
+
known = db.known_job_ids()
|
| 69 |
+
ids = [i for i in store.list_runs() if i not in known]
|
| 70 |
+
if not ids:
|
| 71 |
+
print("[store] no archived runs to restore", flush=True)
|
| 72 |
+
return
|
| 73 |
+
n = 0
|
| 74 |
+
for jid in ids:
|
| 75 |
+
meta = store.fetch_run_meta(jid)
|
| 76 |
+
if meta and db.import_run(meta):
|
| 77 |
+
n += 1
|
| 78 |
+
store._state["restored"] = n
|
| 79 |
+
print(f"[store] restored {n} run(s) from durable storage", flush=True)
|
| 80 |
+
except Exception as exc:
|
| 81 |
+
print(f"[store] history restore failed: {exc!r}", flush=True)
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
threading.Thread(target=_restore_history, name="restore-history",
|
| 85 |
+
daemon=True).start()
|
| 86 |
+
ndata.start_background_build(TOKENIZER)
|
| 87 |
+
worker.start_workers(TOKENIZER_DIR)
|
| 88 |
+
ON_GPU = DEVICES[0].startswith("cuda")
|
| 89 |
+
INFER_DEVICE = DEVICES[0] if ON_GPU else "cpu"
|
| 90 |
+
|
| 91 |
+
app = FastAPI(title="NanoDex")
|
| 92 |
+
IN_SPACE = bool(SPACE_HOST)
|
| 93 |
+
app.add_middleware(
|
| 94 |
+
SessionMiddleware,
|
| 95 |
+
secret_key=os.environ.get("SESSION_SECRET") or (OAUTH_CLIENT_SECRET or secrets.token_hex(32)),
|
| 96 |
+
# A Space is served inside an iframe on huggingface.co, which makes the
|
| 97 |
+
# session cookie third-party: "lax" gets dropped on the redirect back from
|
| 98 |
+
# the OAuth provider and the CSRF state check fails. "none" + Secure is the
|
| 99 |
+
# only combination that survives that round trip.
|
| 100 |
+
same_site="none" if IN_SPACE else "lax",
|
| 101 |
+
https_only=IN_SPACE,
|
| 102 |
+
max_age=8 * 3600,
|
| 103 |
+
)
|
| 104 |
+
app.mount("/static", StaticFiles(directory=os.path.join(BASE, "web/static")), name="static")
|
| 105 |
+
templates = Jinja2Templates(directory=os.path.join(BASE, "web/templates"))
|
| 106 |
+
|
| 107 |
+
# --------------------------------------------------------------- oauth ------
|
| 108 |
+
oauth = None
|
| 109 |
+
if OAUTH_READY:
|
| 110 |
+
from authlib.integrations.starlette_client import OAuth
|
| 111 |
+
oauth = OAuth()
|
| 112 |
+
oauth.register(
|
| 113 |
+
name="huggingface",
|
| 114 |
+
client_id=OAUTH_CLIENT_ID,
|
| 115 |
+
client_secret=OAUTH_CLIENT_SECRET,
|
| 116 |
+
server_metadata_url=f"{OPENID_PROVIDER_URL}/.well-known/openid-configuration",
|
| 117 |
+
client_kwargs={"scope": OAUTH_SCOPES},
|
| 118 |
+
)
|
| 119 |
+
|
| 120 |
+
|
| 121 |
+
def current_user(request: Request) -> Optional[dict]:
|
| 122 |
+
u = request.session.get("user")
|
| 123 |
+
if u and u.get("expires_at", 0) > time.time():
|
| 124 |
+
return u
|
| 125 |
+
if u:
|
| 126 |
+
request.session.pop("user", None)
|
| 127 |
+
return None
|
| 128 |
+
|
| 129 |
+
|
| 130 |
+
def require_user(request: Request) -> dict:
|
| 131 |
+
u = current_user(request)
|
| 132 |
+
if not u:
|
| 133 |
+
raise HTTPException(401, "not signed in")
|
| 134 |
+
return u
|
| 135 |
+
|
| 136 |
+
|
| 137 |
+
def _redirect_uri(request: Request) -> str:
|
| 138 |
+
if SPACE_HOST:
|
| 139 |
+
return f"https://{SPACE_HOST}/auth/callback"
|
| 140 |
+
return str(request.url_for("auth_callback"))
|
| 141 |
+
|
| 142 |
+
|
| 143 |
+
@app.get("/login")
|
| 144 |
+
async def login(request: Request, next: str = "/"):
|
| 145 |
+
request.session["next"] = next
|
| 146 |
+
if not OAUTH_READY:
|
| 147 |
+
# Local development only — never reachable on a Space.
|
| 148 |
+
request.session["user"] = {
|
| 149 |
+
"username": os.environ.get("DEV_USER", "localdev"),
|
| 150 |
+
"name": "Local Dev", "picture": None, "token": None,
|
| 151 |
+
"expires_at": time.time() + 8 * 3600,
|
| 152 |
+
}
|
| 153 |
+
return RedirectResponse(next, status_code=303)
|
| 154 |
+
return await oauth.huggingface.authorize_redirect(request, _redirect_uri(request))
|
| 155 |
+
|
| 156 |
+
|
| 157 |
+
@app.get("/auth/callback", name="auth_callback")
|
| 158 |
+
async def auth_callback(request: Request):
|
| 159 |
+
nxt = request.session.get("next", "/") or "/"
|
| 160 |
+
try:
|
| 161 |
+
token = await oauth.huggingface.authorize_access_token(request)
|
| 162 |
+
except Exception as exc:
|
| 163 |
+
if "state" in str(exc).lower() and request.cookies.get("nx_retry") != "1":
|
| 164 |
+
# The state cookie didn't survive the round trip. Try once more
|
| 165 |
+
# with a fresh session; the marker cookie stops this from looping.
|
| 166 |
+
request.session.clear()
|
| 167 |
+
r = RedirectResponse(f"/login?next={nxt}", status_code=303)
|
| 168 |
+
r.set_cookie("nx_retry", "1", max_age=120, path="/",
|
| 169 |
+
httponly=True, secure=IN_SPACE,
|
| 170 |
+
samesite="none" if IN_SPACE else "lax")
|
| 171 |
+
return r
|
| 172 |
+
return render(request, "error.html", status_code=400,
|
| 173 |
+
code="Sign-in failed", detail=str(exc)[:300])
|
| 174 |
+
info = token.get("userinfo") or await oauth.huggingface.userinfo(token=token)
|
| 175 |
+
request.session["user"] = {
|
| 176 |
+
"username": info.get("preferred_username") or info.get("name"),
|
| 177 |
+
"name": info.get("name") or info.get("preferred_username"),
|
| 178 |
+
"picture": info.get("picture"),
|
| 179 |
+
"token": token.get("access_token"),
|
| 180 |
+
"expires_at": time.time() + 8 * 3600,
|
| 181 |
+
}
|
| 182 |
+
r = RedirectResponse(nxt, status_code=303)
|
| 183 |
+
r.delete_cookie("nx_retry", path="/")
|
| 184 |
+
return r
|
| 185 |
+
|
| 186 |
+
|
| 187 |
+
@app.get("/logout")
|
| 188 |
+
async def logout(request: Request):
|
| 189 |
+
request.session.clear()
|
| 190 |
+
return RedirectResponse("/", status_code=303)
|
| 191 |
+
|
| 192 |
+
|
| 193 |
+
# ---------------------------------------------------------- serialization ---
|
| 194 |
+
def job_dto(j, with_position=False):
|
| 195 |
+
# A row written before a tier list change would otherwise KeyError here and
|
| 196 |
+
# take down the whole page, so fall back to the first tier for display.
|
| 197 |
+
tier = TIERS.get(j["tier"]) or TIERS[TIER_ORDER[0]]
|
| 198 |
+
target = max(int(j["target_tokens"]), 1)
|
| 199 |
+
# Step counts rarely divide the budget exactly, so a finished run would
|
| 200 |
+
# otherwise sit at 99.9% forever.
|
| 201 |
+
progress = (1.0 if j["status"] == "done"
|
| 202 |
+
else min(1.0, (j["tokens_seen"] or 0) / target))
|
| 203 |
+
d = {
|
| 204 |
+
"id": j["id"],
|
| 205 |
+
"name": j["model_name"],
|
| 206 |
+
"username": j["username"],
|
| 207 |
+
"tier": j["tier"],
|
| 208 |
+
"tier_label": tier.label,
|
| 209 |
+
"n_params": j["n_params"],
|
| 210 |
+
"status": j["status"],
|
| 211 |
+
"target_tokens": j["target_tokens"],
|
| 212 |
+
"tokens_seen": j["tokens_seen"] or 0,
|
| 213 |
+
"progress": progress,
|
| 214 |
+
"step": j["step"] or 0,
|
| 215 |
+
"total_steps": j["total_steps"] or 0,
|
| 216 |
+
"loss": j["loss"],
|
| 217 |
+
"best_loss": j["best_loss"],
|
| 218 |
+
"lr": j["lr"],
|
| 219 |
+
"tok_per_s": j["tok_per_s"] or 0,
|
| 220 |
+
"eta_s": j["eta_s"],
|
| 221 |
+
"gpu_index": j["gpu_index"],
|
| 222 |
+
"error": j["error"],
|
| 223 |
+
"pushed_repo": j["pushed_repo"],
|
| 224 |
+
"live_step": j["live_step"],
|
| 225 |
+
"live_at": j["live_at"],
|
| 226 |
+
"has_checkpoint": bool(j["live_step"] is not None) or j["status"] == "done",
|
| 227 |
+
"created_at": j["created_at"],
|
| 228 |
+
"started_at": j["started_at"],
|
| 229 |
+
"finished_at": j["finished_at"],
|
| 230 |
+
"arch": {
|
| 231 |
+
"layers": tier.num_hidden_layers, "hidden": tier.hidden_size,
|
| 232 |
+
"heads": tier.num_attention_heads, "kv_heads": tier.num_key_value_heads,
|
| 233 |
+
"ffn": tier.intermediate_size, "head_dim": tier.head_dim,
|
| 234 |
+
"ctx": SEQ_LEN, "vocab": VOCAB_SIZE,
|
| 235 |
+
},
|
| 236 |
+
}
|
| 237 |
+
if with_position and j["status"] == "queued":
|
| 238 |
+
d["queue_position"] = db.queue_position(j["id"])
|
| 239 |
+
return d
|
| 240 |
+
|
| 241 |
+
|
| 242 |
+
def _tok_label(n):
|
| 243 |
+
return f"{n / 1e9:g}B" if n >= 1_000_000_000 else f"{n / 1e6:g}M"
|
| 244 |
+
|
| 245 |
+
|
| 246 |
+
def tiers_dto():
|
| 247 |
+
out = []
|
| 248 |
+
for k in TIER_ORDER:
|
| 249 |
+
t = TIERS[k]
|
| 250 |
+
out.append({
|
| 251 |
+
"key": k, "label": t.label, "params": t.param_count(),
|
| 252 |
+
"layers": t.num_hidden_layers, "hidden": t.hidden_size,
|
| 253 |
+
"heads": t.num_attention_heads, "kv_heads": t.num_key_value_heads,
|
| 254 |
+
"ffn": t.intermediate_size, "batch_tokens": t.batch_tokens, "lr": t.lr,
|
| 255 |
+
})
|
| 256 |
+
return out
|
| 257 |
+
|
| 258 |
+
|
| 259 |
+
# ------------------------------------------------------------- pages --------
|
| 260 |
+
def render(request: Request, template: str, status_code: int = 200, **kw):
|
| 261 |
+
base = {"user": current_user(request),
|
| 262 |
+
"on_gpu": ON_GPU, "n_devices": len(DEVICES)}
|
| 263 |
+
base.update(kw)
|
| 264 |
+
return templates.TemplateResponse(request, template, base,
|
| 265 |
+
status_code=status_code)
|
| 266 |
+
|
| 267 |
+
|
| 268 |
+
@app.get("/", response_class=HTMLResponse)
|
| 269 |
+
async def index(request: Request):
|
| 270 |
+
u = current_user(request)
|
| 271 |
+
if not u:
|
| 272 |
+
return render(request, "landing.html", stats=db.stats(),
|
| 273 |
+
tiers=tiers_dto(), min_tokens=MIN_TOKENS,
|
| 274 |
+
max_tokens=MAX_TOKENS,
|
| 275 |
+
recent=[job_dto(j) for j in db.list_public(limit=6)])
|
| 276 |
+
jobs = [job_dto(j, True) for j in db.list_jobs(username=u["username"], limit=8)]
|
| 277 |
+
return render(request, "home.html", jobs=jobs, stats=db.stats())
|
| 278 |
+
|
| 279 |
+
|
| 280 |
+
@app.get("/train", response_class=HTMLResponse)
|
| 281 |
+
async def train_page(request: Request):
|
| 282 |
+
if not current_user(request):
|
| 283 |
+
return RedirectResponse("/login?next=/train", status_code=303)
|
| 284 |
+
return render(request, "train.html", tiers=tiers_dto(),
|
| 285 |
+
min_tokens=MIN_TOKENS, max_tokens=MAX_TOKENS,
|
| 286 |
+
stops=TOKEN_STOPS_M,
|
| 287 |
+
max_active=MAX_ACTIVE_PER_USER, seq_len=SEQ_LEN,
|
| 288 |
+
vocab=VOCAB_SIZE)
|
| 289 |
+
|
| 290 |
+
|
| 291 |
+
@app.get("/models", response_class=HTMLResponse)
|
| 292 |
+
async def models_page(request: Request):
|
| 293 |
+
if not current_user(request):
|
| 294 |
+
return RedirectResponse("/login?next=/models", status_code=303)
|
| 295 |
+
return render(request, "models.html")
|
| 296 |
+
|
| 297 |
+
|
| 298 |
+
@app.get("/models/{job_id}", response_class=HTMLResponse)
|
| 299 |
+
async def model_detail(request: Request, job_id: str):
|
| 300 |
+
# Readable by anyone — browsing the shelf shouldn't need a sign-in.
|
| 301 |
+
# Generating, cancelling and publishing still do.
|
| 302 |
+
j = db.get_job(job_id)
|
| 303 |
+
if not j:
|
| 304 |
+
return render(request, "error.html", status_code=404,
|
| 305 |
+
code="404", detail="No run with that id.")
|
| 306 |
+
me = current_user(request)
|
| 307 |
+
return render(request, "detail.html", job=job_dto(j, True),
|
| 308 |
+
infer_device=INFER_DEVICE,
|
| 309 |
+
owner=bool(me and j["username"] == me["username"]))
|
| 310 |
+
|
| 311 |
+
|
| 312 |
+
@app.get("/explore", response_class=HTMLResponse)
|
| 313 |
+
async def explore_page(request: Request):
|
| 314 |
+
return render(request, "explore.html", tiers=tiers_dto())
|
| 315 |
+
|
| 316 |
+
|
| 317 |
+
@app.get("/u/{username}", response_class=HTMLResponse)
|
| 318 |
+
async def public_profile(request: Request, username: str):
|
| 319 |
+
prof = db.user_profile(username)
|
| 320 |
+
if not prof:
|
| 321 |
+
return render(request, "error.html", status_code=404, code="404",
|
| 322 |
+
detail=f"@{username} hasn't trained anything here yet.")
|
| 323 |
+
me = current_user(request)
|
| 324 |
+
return render(request, "user.html", profile=prof, tiers=tiers_dto(),
|
| 325 |
+
is_me=bool(me and me["username"] == username))
|
| 326 |
+
|
| 327 |
+
|
| 328 |
+
@app.get("/queue", response_class=HTMLResponse)
|
| 329 |
+
async def queue_page(request: Request):
|
| 330 |
+
return render(request, "queue.html")
|
| 331 |
+
|
| 332 |
+
|
| 333 |
+
@app.get("/profile", response_class=HTMLResponse)
|
| 334 |
+
async def profile_page(request: Request):
|
| 335 |
+
u = current_user(request)
|
| 336 |
+
if not u:
|
| 337 |
+
return RedirectResponse("/login?next=/profile", status_code=303)
|
| 338 |
+
jobs = db.list_jobs(username=u["username"], limit=500)
|
| 339 |
+
done = [j for j in jobs if j["status"] == "done"]
|
| 340 |
+
return render(request, "profile.html", summary={
|
| 341 |
+
"total": len(jobs), "done": len(done),
|
| 342 |
+
"active": sum(1 for j in jobs if j["status"] in ("queued", "running")),
|
| 343 |
+
"tokens": sum(j["tokens_seen"] or 0 for j in jobs),
|
| 344 |
+
"params": sum(j["n_params"] for j in done),
|
| 345 |
+
"published": sum(1 for j in done if j["pushed_repo"]),
|
| 346 |
+
"best": min([j["best_loss"] for j in done if j["best_loss"]], default=None),
|
| 347 |
+
},
|
| 348 |
+
jobs=[job_dto(j, True) for j in jobs[:20]])
|
| 349 |
+
|
| 350 |
+
|
| 351 |
+
@app.get("/about", response_class=HTMLResponse)
|
| 352 |
+
async def about_page(request: Request):
|
| 353 |
+
return render(request, "about.html", tiers=tiers_dto(),
|
| 354 |
+
min_tokens=MIN_TOKENS, max_tokens=MAX_TOKENS,
|
| 355 |
+
seq_len=SEQ_LEN, vocab=VOCAB_SIZE,
|
| 356 |
+
cache_target=ndata.TARGET_TOKENS,
|
| 357 |
+
max_active=MAX_ACTIVE_PER_USER)
|
| 358 |
+
|
| 359 |
+
|
| 360 |
+
# --------------------------------------------------------------- api --------
|
| 361 |
+
@app.get("/api/me")
|
| 362 |
+
async def api_me(request: Request):
|
| 363 |
+
u = current_user(request)
|
| 364 |
+
if not u:
|
| 365 |
+
return {"signed_in": False}
|
| 366 |
+
return {"signed_in": True, "username": u["username"],
|
| 367 |
+
"name": u["name"], "picture": u["picture"]}
|
| 368 |
+
|
| 369 |
+
|
| 370 |
+
@app.get("/api/tiers")
|
| 371 |
+
async def api_tiers():
|
| 372 |
+
return {"tiers": tiers_dto(), "min_tokens": MIN_TOKENS,
|
| 373 |
+
"max_tokens": MAX_TOKENS, "stops_m": TOKEN_STOPS_M}
|
| 374 |
+
|
| 375 |
+
|
| 376 |
+
@app.get("/api/estimate")
|
| 377 |
+
async def api_estimate(tier: str, tokens: int):
|
| 378 |
+
if tier not in TIERS:
|
| 379 |
+
raise HTTPException(400, "unknown tier")
|
| 380 |
+
tokens = max(MIN_TOKENS, min(MAX_TOKENS, int(tokens)))
|
| 381 |
+
s = estimate.summary(tier, tokens, ON_GPU)
|
| 382 |
+
return {**s, "eta_human": estimate.human_time(s["eta_s"]),
|
| 383 |
+
"chinchilla_x": tokens / (s["n_params"] * 20)}
|
| 384 |
+
|
| 385 |
+
|
| 386 |
+
@app.get("/api/jobs")
|
| 387 |
+
async def api_jobs(request: Request):
|
| 388 |
+
u = require_user(request)
|
| 389 |
+
return {"jobs": [job_dto(j, True)
|
| 390 |
+
for j in db.list_jobs(username=u["username"], limit=200)]}
|
| 391 |
+
|
| 392 |
+
|
| 393 |
+
@app.get("/api/jobs/{job_id}")
|
| 394 |
+
async def api_job(job_id: str):
|
| 395 |
+
j = db.get_job(job_id)
|
| 396 |
+
if not j:
|
| 397 |
+
raise HTTPException(404, "no such run")
|
| 398 |
+
return job_dto(j, True)
|
| 399 |
+
|
| 400 |
+
|
| 401 |
+
@app.get("/api/jobs/{job_id}/metrics")
|
| 402 |
+
async def api_metrics(job_id: str):
|
| 403 |
+
return {"metrics": db.get_metrics(job_id)}
|
| 404 |
+
|
| 405 |
+
|
| 406 |
+
@app.get("/api/jobs/{job_id}/logs")
|
| 407 |
+
async def api_logs(job_id: str):
|
| 408 |
+
return {"logs": db.get_logs(job_id, 300)}
|
| 409 |
+
|
| 410 |
+
|
| 411 |
+
@app.post("/api/jobs")
|
| 412 |
+
async def api_create(request: Request, payload: dict = Body(...)):
|
| 413 |
+
u = require_user(request)
|
| 414 |
+
tier = payload.get("tier")
|
| 415 |
+
if tier not in TIERS:
|
| 416 |
+
raise HTTPException(400, "Pick one of the four model sizes.")
|
| 417 |
+
try:
|
| 418 |
+
tokens = int(payload.get("tokens") or 0)
|
| 419 |
+
except (TypeError, ValueError):
|
| 420 |
+
raise HTTPException(400, "Token budget must be a number.")
|
| 421 |
+
if not (MIN_TOKENS <= tokens <= MAX_TOKENS):
|
| 422 |
+
raise HTTPException(400, f"Token budget must be between "
|
| 423 |
+
f"{_tok_label(MIN_TOKENS)} and {_tok_label(MAX_TOKENS)}.")
|
| 424 |
+
if db.user_active_count(u["username"]) >= MAX_ACTIVE_PER_USER:
|
| 425 |
+
raise HTTPException(429, f"You already have {MAX_ACTIVE_PER_USER} runs in "
|
| 426 |
+
"flight. Wait for one to finish, or cancel it.")
|
| 427 |
+
|
| 428 |
+
import re
|
| 429 |
+
name = re.sub(r"[^A-Za-z0-9._-]+", "-", (payload.get("name") or "").strip()).strip("-._")[:80]
|
| 430 |
+
if not name:
|
| 431 |
+
name = f"{TIERS[tier].label.lower()}-{tokens // 10**6}m"
|
| 432 |
+
|
| 433 |
+
job_id = db.create_job(
|
| 434 |
+
username=u["username"], display_name=u["name"], avatar_url=u["picture"],
|
| 435 |
+
model_name=name, tier=tier, n_params=TIERS[tier].param_count(),
|
| 436 |
+
target_tokens=tokens, total_steps=total_steps_for(tier, tokens),
|
| 437 |
+
seed=int(time.time()) % 100000,
|
| 438 |
+
)
|
| 439 |
+
db.add_log(job_id, f"queued by @{u['username']}")
|
| 440 |
+
return {"id": job_id, "position": db.queue_position(job_id)}
|
| 441 |
+
|
| 442 |
+
|
| 443 |
+
@app.post("/api/jobs/{job_id}/cancel")
|
| 444 |
+
async def api_cancel(request: Request, job_id: str):
|
| 445 |
+
u = require_user(request)
|
| 446 |
+
j = db.get_job(job_id)
|
| 447 |
+
if not j:
|
| 448 |
+
raise HTTPException(404, "no such run")
|
| 449 |
+
if j["username"] != u["username"]:
|
| 450 |
+
raise HTTPException(403, "not your run")
|
| 451 |
+
worker.request_cancel(job_id)
|
| 452 |
+
return {"ok": True}
|
| 453 |
+
|
| 454 |
+
|
| 455 |
+
@app.post("/api/jobs/{job_id}/publish")
|
| 456 |
+
def api_publish(request: Request, job_id: str, payload: dict = Body(default={})):
|
| 457 |
+
u = require_user(request)
|
| 458 |
+
j = db.get_job(job_id)
|
| 459 |
+
if not j:
|
| 460 |
+
raise HTTPException(404, "no such run")
|
| 461 |
+
if j["username"] != u["username"]:
|
| 462 |
+
raise HTTPException(403, "not your run")
|
| 463 |
+
if j["status"] != "done":
|
| 464 |
+
raise HTTPException(400, "This run hasn't finished yet.")
|
| 465 |
+
if not u.get("token"):
|
| 466 |
+
raise HTTPException(401, "Sign in again — your token expired.")
|
| 467 |
+
out_dir = os.path.join(db.RUNS_DIR, job_id)
|
| 468 |
+
if not os.path.exists(os.path.join(out_dir, "model.safetensors")):
|
| 469 |
+
store.fetch_checkpoint(job_id, out_dir)
|
| 470 |
+
if not os.path.exists(os.path.join(out_dir, "model.safetensors")):
|
| 471 |
+
raise HTTPException(410, "Checkpoint could not be restored from storage.")
|
| 472 |
+
import re
|
| 473 |
+
name = re.sub(r"[^A-Za-z0-9._-]+", "-",
|
| 474 |
+
(payload.get("repo") or j["model_name"]).strip()).strip("-._")[:80]
|
| 475 |
+
try:
|
| 476 |
+
repo_id = hub.push_run(out_dir, j, u["token"], name or j["model_name"])
|
| 477 |
+
except Exception as exc:
|
| 478 |
+
raise HTTPException(500, f"Upload failed: {exc}")
|
| 479 |
+
db.update_job(job_id, pushed_repo=repo_id)
|
| 480 |
+
return {"repo": repo_id, "url": f"https://huggingface.co/{repo_id}"}
|
| 481 |
+
|
| 482 |
+
|
| 483 |
+
# One cached inference model, keyed by (job, checkpoint) so a newer snapshot
|
| 484 |
+
# replaces it automatically. Generation runs on the GPU when there is one —
|
| 485 |
+
# a 3M-parameter model costs nothing next to the training job sharing it.
|
| 486 |
+
_pg = {"key": None, "model": None}
|
| 487 |
+
_pg_lock = threading.Lock()
|
| 488 |
+
|
| 489 |
+
|
| 490 |
+
def _resolve_checkpoint(j):
|
| 491 |
+
"""Final weights if the run finished, otherwise the newest live snapshot."""
|
| 492 |
+
final = os.path.join(db.RUNS_DIR, j["id"])
|
| 493 |
+
if j["status"] == "done":
|
| 494 |
+
if not os.path.exists(os.path.join(final, "model.safetensors")):
|
| 495 |
+
# The container was rebuilt since this run finished — pull the
|
| 496 |
+
# weights back out of the bucket.
|
| 497 |
+
store.fetch_checkpoint(j["id"], final)
|
| 498 |
+
if os.path.exists(os.path.join(final, "model.safetensors")):
|
| 499 |
+
return final, ("final", j["step"])
|
| 500 |
+
live = os.path.join(final, "live")
|
| 501 |
+
if os.path.exists(os.path.join(live, "model.safetensors")):
|
| 502 |
+
return live, ("live", j["live_step"])
|
| 503 |
+
return None, (None, None)
|
| 504 |
+
|
| 505 |
+
|
| 506 |
+
@app.post("/api/jobs/{job_id}/generate")
|
| 507 |
+
def api_generate(request: Request, job_id: str, payload: dict = Body(...)):
|
| 508 |
+
# Anyone signed in can generate from anyone's model — that is the point of
|
| 509 |
+
# a shared shelf. Cancelling and publishing stay with the owner.
|
| 510 |
+
require_user(request)
|
| 511 |
+
j = db.get_job(job_id)
|
| 512 |
+
if not j:
|
| 513 |
+
raise HTTPException(404, "no such run")
|
| 514 |
+
|
| 515 |
+
path, (kind, step) = _resolve_checkpoint(j)
|
| 516 |
+
if not path:
|
| 517 |
+
if j["status"] == "queued":
|
| 518 |
+
raise HTTPException(409, "This run hasn't started yet — "
|
| 519 |
+
"the first checkpoint appears a few steps in.")
|
| 520 |
+
if j["status"] == "running":
|
| 521 |
+
raise HTTPException(409, "No checkpoint saved yet. The first one lands "
|
| 522 |
+
"within the first few percent of the run.")
|
| 523 |
+
raise HTTPException(410, "Checkpoint is gone — the Space restarted.")
|
| 524 |
+
|
| 525 |
+
import torch
|
| 526 |
+
from transformers import AutoModelForCausalLM
|
| 527 |
+
|
| 528 |
+
key = (path, step)
|
| 529 |
+
with _pg_lock:
|
| 530 |
+
if _pg["key"] != key:
|
| 531 |
+
m = AutoModelForCausalLM.from_pretrained(path, dtype=torch.float32)
|
| 532 |
+
m.config.use_cache = True
|
| 533 |
+
_pg["model"] = m.eval().to(INFER_DEVICE)
|
| 534 |
+
_pg["key"] = key
|
| 535 |
+
model = _pg["model"]
|
| 536 |
+
|
| 537 |
+
prompt = payload.get("prompt") or " "
|
| 538 |
+
ids = TOKENIZER.encode(prompt).ids[-(SEQ_LEN - 1):] or [0]
|
| 539 |
+
temp = float(payload.get("temperature", 0.85))
|
| 540 |
+
x = torch.tensor([ids], dtype=torch.long, device=INFER_DEVICE)
|
| 541 |
+
t0 = time.time()
|
| 542 |
+
with torch.no_grad():
|
| 543 |
+
out = model.generate(
|
| 544 |
+
x,
|
| 545 |
+
max_new_tokens=max(1, min(int(payload.get("max_new_tokens", 80)), 256)),
|
| 546 |
+
do_sample=temp > 0.01, temperature=max(temp, 1e-4),
|
| 547 |
+
top_k=int(payload.get("top_k", 40)),
|
| 548 |
+
top_p=float(payload.get("top_p", 0.95)),
|
| 549 |
+
pad_token_id=1, eos_token_id=None,
|
| 550 |
+
)
|
| 551 |
+
ids_out = out[0].tolist()
|
| 552 |
+
|
| 553 |
+
return {
|
| 554 |
+
"text": TOKENIZER.decode(ids_out),
|
| 555 |
+
"prompt": prompt,
|
| 556 |
+
"continuation": TOKENIZER.decode(ids_out[len(ids):]),
|
| 557 |
+
"checkpoint": kind,
|
| 558 |
+
"step": step,
|
| 559 |
+
"total_steps": j["total_steps"],
|
| 560 |
+
"device": INFER_DEVICE,
|
| 561 |
+
"ms": round((time.time() - t0) * 1000),
|
| 562 |
+
"new_tokens": len(ids_out) - len(ids),
|
| 563 |
+
}
|
| 564 |
+
|
| 565 |
+
|
| 566 |
+
@app.get("/api/explore")
|
| 567 |
+
async def api_explore(tier: str = "", sort: str = "recent",
|
| 568 |
+
limit: int = 60, offset: int = 0, username: str = ""):
|
| 569 |
+
limit = max(1, min(limit, 120))
|
| 570 |
+
rows = db.list_public(tier=tier or None, sort=sort, limit=limit,
|
| 571 |
+
offset=max(0, offset), username=username or None)
|
| 572 |
+
return {"models": [job_dto(j) for j in rows],
|
| 573 |
+
"active": [job_dto(j, True) for j in db.list_active_public()],
|
| 574 |
+
"stats": db.stats()}
|
| 575 |
+
|
| 576 |
+
|
| 577 |
+
@app.get("/api/leaderboard")
|
| 578 |
+
async def api_leaderboard():
|
| 579 |
+
lb = db.leaderboard()
|
| 580 |
+
return {"tiers": [
|
| 581 |
+
{"tier": k, "label": (TIERS.get(k) or TIERS[TIER_ORDER[0]]).label,
|
| 582 |
+
"order": TIER_ORDER.index(k) if k in TIER_ORDER else 99,
|
| 583 |
+
"models": [job_dto(j) for j in v]}
|
| 584 |
+
for k, v in sorted(lb.items(),
|
| 585 |
+
key=lambda kv: TIER_ORDER.index(kv[0])
|
| 586 |
+
if kv[0] in TIER_ORDER else 99)]}
|
| 587 |
+
|
| 588 |
+
|
| 589 |
+
@app.get("/api/users/{username}")
|
| 590 |
+
async def api_user(username: str):
|
| 591 |
+
prof = db.user_profile(username)
|
| 592 |
+
if not prof:
|
| 593 |
+
raise HTTPException(404, "no such trainer")
|
| 594 |
+
return {"profile": prof,
|
| 595 |
+
"models": [job_dto(j) for j in
|
| 596 |
+
db.list_public(username=username, limit=100)],
|
| 597 |
+
"active": [job_dto(j, True) for j in
|
| 598 |
+
db.list_jobs(username=username, limit=100)
|
| 599 |
+
if j["status"] in ("queued", "running")]}
|
| 600 |
+
|
| 601 |
+
|
| 602 |
+
@app.get("/api/trainers")
|
| 603 |
+
async def api_trainers():
|
| 604 |
+
return {"trainers": db.top_trainers()}
|
| 605 |
+
|
| 606 |
+
|
| 607 |
+
@app.get("/api/queue")
|
| 608 |
+
async def api_queue():
|
| 609 |
+
infos = worker.worker_info()
|
| 610 |
+
return {
|
| 611 |
+
"workers": [{"index": w["index"], "device": w["device"],
|
| 612 |
+
"job": job_dto(w["job"]) if w["job"] else None}
|
| 613 |
+
for w in infos],
|
| 614 |
+
"running": [job_dto(j) for j in db.list_jobs(status=["running"], limit=64)],
|
| 615 |
+
"queued": [job_dto(j, True) for j in db.list_jobs(status=["queued"], limit=64)],
|
| 616 |
+
"recent": [job_dto(j) for j in db.list_jobs(status=["done"], limit=12)],
|
| 617 |
+
"stats": db.stats(),
|
| 618 |
+
"data": {**ndata.state(), "target": ndata.TARGET_TOKENS},
|
| 619 |
+
"on_gpu": ON_GPU,
|
| 620 |
+
"store": {**store.state(), "bucket": store.BUCKET_ID},
|
| 621 |
+
}
|
| 622 |
+
|
| 623 |
+
|
| 624 |
+
@app.get("/api/health")
|
| 625 |
+
async def health():
|
| 626 |
+
return {"ok": True, "devices": DEVICES, "data": ndata.state(),
|
| 627 |
+
"stats": db.stats(), "store": store.state(),
|
| 628 |
+
"bucket": store.BUCKET_ID}
|
| 629 |
+
|
| 630 |
+
|
| 631 |
+
@app.exception_handler(HTTPException)
|
| 632 |
+
async def http_error(request: Request, exc: HTTPException):
|
| 633 |
+
if request.url.path.startswith("/api/"):
|
| 634 |
+
return JSONResponse({"error": exc.detail}, status_code=exc.status_code)
|
| 635 |
+
if exc.status_code == 401:
|
| 636 |
+
return RedirectResponse(f"/login?next={request.url.path}", status_code=303)
|
| 637 |
+
return render(request, "error.html", status_code=exc.status_code,
|
| 638 |
+
code=str(exc.status_code), detail=exc.detail)
|
| 639 |
+
|
| 640 |
+
|
| 641 |
+
if __name__ == "__main__":
|
| 642 |
+
import uvicorn
|
| 643 |
+
uvicorn.run(app, host="0.0.0.0", port=int(os.environ.get("PORT", 7860)))
|
nanodex/__init__.py
ADDED
|
File without changes
|
nanodex/config.py
ADDED
|
@@ -0,0 +1,208 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Model tier definitions for NanoDex — small decoder-only language models
|
| 3 |
+
trained from scratch.
|
| 4 |
+
|
| 5 |
+
All four tiers share one standard modern transformer recipe, scaled down
|
| 6 |
+
in width and depth:
|
| 7 |
+
- LlamaForCausalLM, SiLU MLP, RMSNorm (eps 1e-5), RoPE theta 10000
|
| 8 |
+
- Grouped-query attention, no attention/MLP biases
|
| 9 |
+
- Tied input/output embeddings
|
| 10 |
+
|
| 11 |
+
A typical small-LM vocabulary (~49k tokens) would be ~28M embedding
|
| 12 |
+
parameters on its own, which makes a 500k-parameter model impossible.
|
| 13 |
+
NanoDex therefore uses a purpose-built 2,048-token BPE trained on
|
| 14 |
+
fineweb-edu, so the advertised size is the REAL total parameter count
|
| 15 |
+
(embeddings included), not "non-embedding params".
|
| 16 |
+
"""
|
| 17 |
+
|
| 18 |
+
from dataclasses import dataclass
|
| 19 |
+
|
| 20 |
+
VOCAB_SIZE = 2048
|
| 21 |
+
MAX_POSITION_EMBEDDINGS = 512
|
| 22 |
+
SEQ_LEN = 512
|
| 23 |
+
RMS_NORM_EPS = 1e-5
|
| 24 |
+
ROPE_THETA = 10000.0
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
@dataclass
|
| 28 |
+
class Tier:
|
| 29 |
+
key: str
|
| 30 |
+
label: str
|
| 31 |
+
hidden_size: int
|
| 32 |
+
num_hidden_layers: int
|
| 33 |
+
num_attention_heads: int
|
| 34 |
+
num_key_value_heads: int
|
| 35 |
+
intermediate_size: int
|
| 36 |
+
# Peak learning rate. These follow LR * hidden_size = 0.384, the muP-style
|
| 37 |
+
# rule that keeps update sizes comparable across widths. The anchor was
|
| 38 |
+
# checked by sweeping the 500k tier (1e-3 through 1.6e-2); 4e-3 won.
|
| 39 |
+
lr: float
|
| 40 |
+
batch_tokens: int # tokens per optimizer step (global)
|
| 41 |
+
micro_batch: int # sequences per forward pass, per GPU
|
| 42 |
+
# (halved automatically if CUDA OOMs)
|
| 43 |
+
|
| 44 |
+
@property
|
| 45 |
+
def head_dim(self) -> int:
|
| 46 |
+
return self.hidden_size // self.num_attention_heads
|
| 47 |
+
|
| 48 |
+
def param_count(self, vocab_size: int = VOCAB_SIZE) -> int:
|
| 49 |
+
h = self.hidden_size
|
| 50 |
+
kv = self.num_key_value_heads * self.head_dim
|
| 51 |
+
per_layer = (
|
| 52 |
+
h * h # q_proj
|
| 53 |
+
+ h * kv # k_proj
|
| 54 |
+
+ h * kv # v_proj
|
| 55 |
+
+ h * h # o_proj
|
| 56 |
+
+ 3 * h * self.intermediate_size # gate, up, down
|
| 57 |
+
+ 2 * h # input_layernorm + post_attention_layernorm
|
| 58 |
+
)
|
| 59 |
+
return vocab_size * h + self.num_hidden_layers * per_layer + h
|
| 60 |
+
|
| 61 |
+
def to_hf_config_dict(self, vocab_size: int = VOCAB_SIZE) -> dict:
|
| 62 |
+
return {
|
| 63 |
+
"architectures": ["LlamaForCausalLM"],
|
| 64 |
+
"attention_bias": False,
|
| 65 |
+
"attention_dropout": 0.0,
|
| 66 |
+
"bos_token_id": 0,
|
| 67 |
+
"eos_token_id": 0,
|
| 68 |
+
"pad_token_id": 1,
|
| 69 |
+
"hidden_act": "silu",
|
| 70 |
+
"hidden_size": self.hidden_size,
|
| 71 |
+
"initializer_range": 0.041666666666666664,
|
| 72 |
+
"intermediate_size": self.intermediate_size,
|
| 73 |
+
"max_position_embeddings": MAX_POSITION_EMBEDDINGS,
|
| 74 |
+
"mlp_bias": False,
|
| 75 |
+
"model_type": "llama",
|
| 76 |
+
"num_attention_heads": self.num_attention_heads,
|
| 77 |
+
"num_hidden_layers": self.num_hidden_layers,
|
| 78 |
+
"num_key_value_heads": self.num_key_value_heads,
|
| 79 |
+
"pretraining_tp": 1,
|
| 80 |
+
"rms_norm_eps": RMS_NORM_EPS,
|
| 81 |
+
"rope_scaling": None,
|
| 82 |
+
"rope_theta": ROPE_THETA,
|
| 83 |
+
"tie_word_embeddings": True,
|
| 84 |
+
"torch_dtype": "float32",
|
| 85 |
+
"use_cache": True,
|
| 86 |
+
"vocab_size": vocab_size,
|
| 87 |
+
}
|
| 88 |
+
|
| 89 |
+
TIERS = {
|
| 90 |
+
"100k": Tier(
|
| 91 |
+
key="100k", label="NanoDex-100K",
|
| 92 |
+
hidden_size=32, num_hidden_layers=2,
|
| 93 |
+
num_attention_heads=2, num_key_value_heads=1,
|
| 94 |
+
intermediate_size=147,
|
| 95 |
+
lr=5e-3, batch_tokens=65536, micro_batch=128,
|
| 96 |
+
),
|
| 97 |
+
"250k": Tier(
|
| 98 |
+
key="250k", label="NanoDex-250K",
|
| 99 |
+
hidden_size=48, num_hidden_layers=4,
|
| 100 |
+
num_attention_heads=4, num_key_value_heads=2,
|
| 101 |
+
intermediate_size=215,
|
| 102 |
+
lr=4.5e-3, batch_tokens=65536, micro_batch=128,
|
| 103 |
+
),
|
| 104 |
+
"500k": Tier(
|
| 105 |
+
key="500k", label="NanoDex-500K",
|
| 106 |
+
hidden_size=80, num_hidden_layers=4,
|
| 107 |
+
num_attention_heads=5, num_key_value_heads=1,
|
| 108 |
+
intermediate_size=285,
|
| 109 |
+
lr=4e-3, batch_tokens=131072, micro_batch=256,
|
| 110 |
+
),
|
| 111 |
+
"1m": Tier(
|
| 112 |
+
key="1m", label="NanoDex-1M",
|
| 113 |
+
hidden_size=96, num_hidden_layers=5,
|
| 114 |
+
num_attention_heads=6, num_key_value_heads=2,
|
| 115 |
+
intermediate_size=472,
|
| 116 |
+
lr=3e-3, batch_tokens=131072, micro_batch=256,
|
| 117 |
+
),
|
| 118 |
+
"2m": Tier(
|
| 119 |
+
key="2m", label="NanoDex-2M",
|
| 120 |
+
hidden_size=128, num_hidden_layers=6,
|
| 121 |
+
num_attention_heads=8, num_key_value_heads=2,
|
| 122 |
+
intermediate_size=647,
|
| 123 |
+
lr=2.5e-3, batch_tokens=262144, micro_batch=256,
|
| 124 |
+
),
|
| 125 |
+
"5m": Tier(
|
| 126 |
+
key="5m", label="NanoDex-5M",
|
| 127 |
+
hidden_size=192, num_hidden_layers=8,
|
| 128 |
+
num_attention_heads=8, num_key_value_heads=2,
|
| 129 |
+
intermediate_size=839,
|
| 130 |
+
lr=1.5e-3, batch_tokens=393216, micro_batch=128,
|
| 131 |
+
),
|
| 132 |
+
"10m": Tier(
|
| 133 |
+
key="10m", label="NanoDex-10M",
|
| 134 |
+
hidden_size=256, num_hidden_layers=10,
|
| 135 |
+
num_attention_heads=8, num_key_value_heads=2,
|
| 136 |
+
intermediate_size=1020,
|
| 137 |
+
lr=1.2e-3, batch_tokens=524288, micro_batch=128,
|
| 138 |
+
),
|
| 139 |
+
"25m": Tier(
|
| 140 |
+
key="25m", label="NanoDex-25M",
|
| 141 |
+
hidden_size=320, num_hidden_layers=12,
|
| 142 |
+
num_attention_heads=10, num_key_value_heads=2,
|
| 143 |
+
intermediate_size=1856,
|
| 144 |
+
lr=8e-4, batch_tokens=524288, micro_batch=64,
|
| 145 |
+
),
|
| 146 |
+
"50m": Tier(
|
| 147 |
+
key="50m", label="NanoDex-50M",
|
| 148 |
+
hidden_size=448, num_hidden_layers=12,
|
| 149 |
+
num_attention_heads=8, num_key_value_heads=2,
|
| 150 |
+
intermediate_size=2669,
|
| 151 |
+
lr=6e-4, batch_tokens=524288, micro_batch=32,
|
| 152 |
+
),
|
| 153 |
+
"100m": Tier(
|
| 154 |
+
key="100m", label="NanoDex-100M",
|
| 155 |
+
hidden_size=640, num_hidden_layers=21,
|
| 156 |
+
num_attention_heads=8, num_key_value_heads=2,
|
| 157 |
+
intermediate_size=1792,
|
| 158 |
+
lr=4e-4, batch_tokens=1048576, micro_batch=32,
|
| 159 |
+
),
|
| 160 |
+
"gpt2-small": Tier(
|
| 161 |
+
key="gpt2-small", label="GPT-2 Small",
|
| 162 |
+
hidden_size=768, num_hidden_layers=12,
|
| 163 |
+
num_attention_heads=12, num_key_value_heads=12,
|
| 164 |
+
intermediate_size=3072,
|
| 165 |
+
lr=3e-4, batch_tokens=524288, micro_batch=16,
|
| 166 |
+
),
|
| 167 |
+
}
|
| 168 |
+
|
| 169 |
+
TIER_ORDER = [
|
| 170 |
+
"100k",
|
| 171 |
+
"250k",
|
| 172 |
+
"500k",
|
| 173 |
+
"1m",
|
| 174 |
+
"2m",
|
| 175 |
+
"5m",
|
| 176 |
+
"10m",
|
| 177 |
+
"25m",
|
| 178 |
+
"50m",
|
| 179 |
+
"100m",
|
| 180 |
+
"gpt2-small"
|
| 181 |
+
]
|
| 182 |
+
|
| 183 |
+
MIN_TOKENS = 1_000_000
|
| 184 |
+
MAX_TOKENS = 100_000_000_000
|
| 185 |
+
|
| 186 |
+
TOKEN_STOPS_M = [
|
| 187 |
+
1, 5, 10, 25, 50, 100, 200, 300, 400, 500,
|
| 188 |
+
750, 1000, 1500, 2000, 3000,
|
| 189 |
+
4000, 5000, 6000, 7500, 8000,
|
| 190 |
+
9000, 10000, 25000, 50000, 100000
|
| 191 |
+
]
|
| 192 |
+
|
| 193 |
+
def tier_or_none(key):
|
| 194 |
+
"""Tolerant lookup — old rows may reference tiers that no longer exist."""
|
| 195 |
+
return TIERS.get(key)
|
| 196 |
+
|
| 197 |
+
|
| 198 |
+
if __name__ == "__main__":
|
| 199 |
+
for k in TIER_ORDER:
|
| 200 |
+
t = TIERS[k]
|
| 201 |
+
p = t.param_count()
|
| 202 |
+
tpm = t.micro_batch * SEQ_LEN
|
| 203 |
+
accum = max(1, t.batch_tokens // tpm)
|
| 204 |
+
exact = "exact" if t.micro_batch * SEQ_LEN * accum == t.batch_tokens else "MISALIGNED"
|
| 205 |
+
print(f"{t.label:14s} {p:>10,d} h={t.hidden_size:3d} L={t.num_hidden_layers:2d} "
|
| 206 |
+
f"heads={t.num_attention_heads}/{t.num_key_value_heads} hd={t.head_dim} "
|
| 207 |
+
f"ffn={t.intermediate_size:3d} | micro={t.micro_batch} accum={accum} "
|
| 208 |
+
f"({t.batch_tokens:,} tok/step, {exact}) lr={t.lr:.1e}")
|
nanodex/data.py
ADDED
|
@@ -0,0 +1,282 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Shared fineweb-edu token cache.
|
| 3 |
+
|
| 4 |
+
Every job trains on the same corpus (HuggingFaceFW/fineweb-edu), so instead of
|
| 5 |
+
each run streaming and re-tokenizing the dataset, the Space builds ONE flat
|
| 6 |
+
uint16 token file in a background thread at startup and all trainers read
|
| 7 |
+
slices of it. 300M tokens x 2 bytes = 600 MB on disk, which comfortably fits
|
| 8 |
+
Spaces' ephemeral storage and makes the GPUs data-bound never again.
|
| 9 |
+
|
| 10 |
+
Documents are packed end-to-end separated by <|endoftext|> (id 0).
|
| 11 |
+
"""
|
| 12 |
+
|
| 13 |
+
import os
|
| 14 |
+
import threading
|
| 15 |
+
import time
|
| 16 |
+
|
| 17 |
+
import numpy as np
|
| 18 |
+
|
| 19 |
+
from . import store
|
| 20 |
+
from .config import MAX_TOKENS
|
| 21 |
+
from .db import DATA_DIR
|
| 22 |
+
|
| 23 |
+
TOKENS_PATH = os.path.join(DATA_DIR, "fineweb_edu_tokens.u16")
|
| 24 |
+
META_PATH = TOKENS_PATH + ".meta"
|
| 25 |
+
|
| 26 |
+
DATASET_ID = "HuggingFaceFW/fineweb-edu"
|
| 27 |
+
DATASET_CONFIG = "sample-350BT"
|
| 28 |
+
|
| 29 |
+
# Build a little past the maximum a user can request so the last run never
|
| 30 |
+
# stalls on the writer. At 2 bytes per token a 1B-token budget is ~2 GB on
|
| 31 |
+
# disk, which is nothing next to the Space's ephemeral storage.
|
| 32 |
+
TARGET_TOKENS = MAX_TOKENS + 5_000_000
|
| 33 |
+
|
| 34 |
+
_state = {
|
| 35 |
+
"ready_tokens": 0,
|
| 36 |
+
"status": "idle", # idle|building|ready|error
|
| 37 |
+
"error": None,
|
| 38 |
+
"started": None,
|
| 39 |
+
"rate": 0.0,
|
| 40 |
+
}
|
| 41 |
+
_state_lock = threading.Lock()
|
| 42 |
+
_build_thread = None
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
def state():
|
| 46 |
+
with _state_lock:
|
| 47 |
+
return dict(_state)
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def _set(**kw):
|
| 51 |
+
with _state_lock:
|
| 52 |
+
_state.update(kw)
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def _load_meta():
|
| 56 |
+
if os.path.exists(META_PATH):
|
| 57 |
+
try:
|
| 58 |
+
with open(META_PATH) as f:
|
| 59 |
+
return int(f.read().strip())
|
| 60 |
+
except Exception:
|
| 61 |
+
return 0
|
| 62 |
+
return 0
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
def _save_meta(n):
|
| 66 |
+
tmp = META_PATH + ".tmp"
|
| 67 |
+
with open(tmp, "w") as f:
|
| 68 |
+
f.write(str(n))
|
| 69 |
+
os.replace(tmp, META_PATH)
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
def build_token_cache(tokenizer, log=print):
|
| 73 |
+
"""Stream fineweb-edu, tokenize, append to the flat uint16 file."""
|
| 74 |
+
from datasets import load_dataset
|
| 75 |
+
|
| 76 |
+
# A previous container may have already paid for this. Downloading ~2.8 GB
|
| 77 |
+
# beats re-tokenizing 1.5B tokens by a wide margin.
|
| 78 |
+
have = _load_meta()
|
| 79 |
+
if have < TARGET_TOKENS:
|
| 80 |
+
restored = store.fetch_token_cache(TOKENS_PATH, META_PATH, log)
|
| 81 |
+
if restored > have:
|
| 82 |
+
have = restored
|
| 83 |
+
_set(ready_tokens=have)
|
| 84 |
+
|
| 85 |
+
if have >= TARGET_TOKENS:
|
| 86 |
+
_set(ready_tokens=have, status="ready")
|
| 87 |
+
log(f"[data] cache already complete: {have:,} tokens")
|
| 88 |
+
return
|
| 89 |
+
|
| 90 |
+
_set(status="building", started=time.time(), ready_tokens=have)
|
| 91 |
+
log(f"[data] building token cache ({have:,} / {TARGET_TOKENS:,})")
|
| 92 |
+
|
| 93 |
+
eot = 0 # <|endoftext|>
|
| 94 |
+
t0 = time.time()
|
| 95 |
+
written = have
|
| 96 |
+
|
| 97 |
+
# Truncate to the last known-good offset, then append.
|
| 98 |
+
mode = "r+b" if os.path.exists(TOKENS_PATH) else "wb"
|
| 99 |
+
f = open(TOKENS_PATH, mode)
|
| 100 |
+
f.seek(written * 2)
|
| 101 |
+
f.truncate()
|
| 102 |
+
|
| 103 |
+
try:
|
| 104 |
+
ds = load_dataset(DATASET_ID, name=DATASET_CONFIG,
|
| 105 |
+
split="train", streaming=True)
|
| 106 |
+
# Skip ahead deterministically if resuming a partial build.
|
| 107 |
+
buf_docs = []
|
| 108 |
+
BATCH = 512
|
| 109 |
+
for row in ds:
|
| 110 |
+
buf_docs.append(row["text"])
|
| 111 |
+
if len(buf_docs) < BATCH:
|
| 112 |
+
continue
|
| 113 |
+
encs = tokenizer.encode_batch(buf_docs)
|
| 114 |
+
buf_docs = []
|
| 115 |
+
flat = []
|
| 116 |
+
for e in encs:
|
| 117 |
+
flat.extend(e.ids)
|
| 118 |
+
flat.append(eot)
|
| 119 |
+
arr = np.asarray(flat, dtype=np.uint16)
|
| 120 |
+
f.write(arr.tobytes())
|
| 121 |
+
written += arr.size
|
| 122 |
+
if written % (1 << 22) < arr.size: # ~every 4M tokens
|
| 123 |
+
f.flush()
|
| 124 |
+
_save_meta(written)
|
| 125 |
+
_touch_lock()
|
| 126 |
+
dt = max(time.time() - t0, 1e-6)
|
| 127 |
+
_set(ready_tokens=written, rate=(written - have) / dt)
|
| 128 |
+
log(f"[data] {written:,} tokens "
|
| 129 |
+
f"({(written - have) / dt / 1e6:.2f}M tok/s)")
|
| 130 |
+
if written >= TARGET_TOKENS:
|
| 131 |
+
break
|
| 132 |
+
f.flush()
|
| 133 |
+
_save_meta(written)
|
| 134 |
+
_set(ready_tokens=written, status="ready")
|
| 135 |
+
log(f"[data] cache ready: {written:,} tokens in {time.time() - t0:.0f}s")
|
| 136 |
+
f.close()
|
| 137 |
+
store.save_token_cache(TOKENS_PATH, META_PATH, log)
|
| 138 |
+
except Exception as exc: # keep the Space alive; jobs will wait / use what exists
|
| 139 |
+
_save_meta(written)
|
| 140 |
+
_set(ready_tokens=written, status="error", error=repr(exc))
|
| 141 |
+
log(f"[data] ERROR: {exc!r} — the partial cache of {written:,} tokens is still usable")
|
| 142 |
+
finally:
|
| 143 |
+
f.close()
|
| 144 |
+
|
| 145 |
+
|
| 146 |
+
LOCK_PATH = TOKENS_PATH + ".lock"
|
| 147 |
+
LOCK_STALE_SECONDS = 150
|
| 148 |
+
_HEARTBEAT = 30
|
| 149 |
+
|
| 150 |
+
|
| 151 |
+
def _acquire_build_lock():
|
| 152 |
+
"""
|
| 153 |
+
Only one process may append to the shared token file. A rolling deploy runs
|
| 154 |
+
two containers against the same volume for a while, and two appenders would
|
| 155 |
+
corrupt it. The lock is a file the holder keeps touching, so a crashed
|
| 156 |
+
holder's lock goes stale instead of blocking forever.
|
| 157 |
+
"""
|
| 158 |
+
now = time.time()
|
| 159 |
+
try:
|
| 160 |
+
fd = os.open(LOCK_PATH, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o644)
|
| 161 |
+
os.write(fd, str(os.getpid()).encode())
|
| 162 |
+
os.close(fd)
|
| 163 |
+
return True
|
| 164 |
+
except FileExistsError:
|
| 165 |
+
try:
|
| 166 |
+
age = now - os.path.getmtime(LOCK_PATH)
|
| 167 |
+
except OSError:
|
| 168 |
+
return False
|
| 169 |
+
if age < LOCK_STALE_SECONDS:
|
| 170 |
+
return False
|
| 171 |
+
# Stale: the previous holder is gone. Take it over.
|
| 172 |
+
try:
|
| 173 |
+
os.utime(LOCK_PATH, (now, now))
|
| 174 |
+
with open(LOCK_PATH, "w") as f:
|
| 175 |
+
f.write(str(os.getpid()))
|
| 176 |
+
return True
|
| 177 |
+
except OSError:
|
| 178 |
+
return False
|
| 179 |
+
except OSError:
|
| 180 |
+
return True # can't lock (read-only fs) — proceed anyway
|
| 181 |
+
|
| 182 |
+
|
| 183 |
+
def _touch_lock():
|
| 184 |
+
try:
|
| 185 |
+
os.utime(LOCK_PATH, None)
|
| 186 |
+
except OSError:
|
| 187 |
+
pass
|
| 188 |
+
|
| 189 |
+
|
| 190 |
+
def _release_lock():
|
| 191 |
+
try:
|
| 192 |
+
os.unlink(LOCK_PATH)
|
| 193 |
+
except OSError:
|
| 194 |
+
pass
|
| 195 |
+
|
| 196 |
+
|
| 197 |
+
def _guarded_build(tokenizer, log):
|
| 198 |
+
if not _acquire_build_lock():
|
| 199 |
+
log("[data] another process is building the token cache — "
|
| 200 |
+
"using it read-only")
|
| 201 |
+
_set(status="following")
|
| 202 |
+
# Track the other process's progress so jobs know what's available.
|
| 203 |
+
while True:
|
| 204 |
+
have = _load_meta()
|
| 205 |
+
_set(ready_tokens=have)
|
| 206 |
+
if have >= TARGET_TOKENS:
|
| 207 |
+
_set(status="ready")
|
| 208 |
+
return
|
| 209 |
+
try:
|
| 210 |
+
if time.time() - os.path.getmtime(LOCK_PATH) > LOCK_STALE_SECONDS:
|
| 211 |
+
break # holder died; take over below
|
| 212 |
+
except OSError:
|
| 213 |
+
break
|
| 214 |
+
time.sleep(10)
|
| 215 |
+
if not _acquire_build_lock():
|
| 216 |
+
_set(status="ready" if _load_meta() else "error")
|
| 217 |
+
return
|
| 218 |
+
try:
|
| 219 |
+
build_token_cache(tokenizer, log)
|
| 220 |
+
finally:
|
| 221 |
+
_release_lock()
|
| 222 |
+
|
| 223 |
+
|
| 224 |
+
def start_background_build(tokenizer, log=print):
|
| 225 |
+
global _build_thread
|
| 226 |
+
if _build_thread and _build_thread.is_alive():
|
| 227 |
+
return _build_thread
|
| 228 |
+
_build_thread = threading.Thread(
|
| 229 |
+
target=_guarded_build, args=(tokenizer, log),
|
| 230 |
+
name="token-cache", daemon=True)
|
| 231 |
+
_build_thread.start()
|
| 232 |
+
return _build_thread
|
| 233 |
+
|
| 234 |
+
|
| 235 |
+
def available_tokens():
|
| 236 |
+
with _state_lock:
|
| 237 |
+
return _state["ready_tokens"]
|
| 238 |
+
|
| 239 |
+
|
| 240 |
+
class TokenStream:
|
| 241 |
+
"""
|
| 242 |
+
Sequential reader over the shared token file, yielding (B, T) int64
|
| 243 |
+
batches. Each job gets its own random start offset so two runs of the same
|
| 244 |
+
size do not see the identical token order.
|
| 245 |
+
|
| 246 |
+
The batch is used as BOTH input_ids and labels: LlamaForCausalLM shifts
|
| 247 |
+
labels internally, so handing it a pre-shifted target would train the model
|
| 248 |
+
to predict two tokens ahead while generation samples one ahead.
|
| 249 |
+
"""
|
| 250 |
+
|
| 251 |
+
def __init__(self, seq_len, batch_size, seed=1337):
|
| 252 |
+
self.seq_len = seq_len
|
| 253 |
+
self.batch_size = batch_size
|
| 254 |
+
self.chunk = batch_size * seq_len
|
| 255 |
+
rng = np.random.default_rng(seed)
|
| 256 |
+
avail = max(available_tokens(), self.chunk + 1)
|
| 257 |
+
self.pos = int(rng.integers(0, max(avail - self.chunk, 1)))
|
| 258 |
+
self._mm = None
|
| 259 |
+
self._mm_len = 0
|
| 260 |
+
|
| 261 |
+
def _ensure(self):
|
| 262 |
+
n = available_tokens()
|
| 263 |
+
if self._mm is None or n > self._mm_len:
|
| 264 |
+
if n < self.chunk + 1:
|
| 265 |
+
return False
|
| 266 |
+
self._mm = np.memmap(TOKENS_PATH, dtype=np.uint16, mode="r", shape=(n,))
|
| 267 |
+
self._mm_len = n
|
| 268 |
+
return self._mm is not None
|
| 269 |
+
|
| 270 |
+
def next_batch(self):
|
| 271 |
+
"""One (B, T) int64 batch. Pass it as input_ids AND labels."""
|
| 272 |
+
import torch
|
| 273 |
+
deadline = time.time() + 300
|
| 274 |
+
while not self._ensure():
|
| 275 |
+
if time.time() > deadline:
|
| 276 |
+
raise RuntimeError("token cache unavailable (timed out waiting for data)")
|
| 277 |
+
time.sleep(2)
|
| 278 |
+
if self.pos + self.chunk > self._mm_len:
|
| 279 |
+
self.pos = 0
|
| 280 |
+
buf = np.asarray(self._mm[self.pos:self.pos + self.chunk], dtype=np.int64)
|
| 281 |
+
self.pos += self.chunk
|
| 282 |
+
return torch.from_numpy(buf.reshape(self.batch_size, self.seq_len))
|
nanodex/db.py
ADDED
|
@@ -0,0 +1,370 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""SQLite-backed job store for the NanoDex training queue."""
|
| 2 |
+
|
| 3 |
+
import json
|
| 4 |
+
import os
|
| 5 |
+
import sqlite3
|
| 6 |
+
import threading
|
| 7 |
+
import time
|
| 8 |
+
import uuid
|
| 9 |
+
|
| 10 |
+
# Spaces persistent storage mounts at /data. Fall back to a local dir.
|
| 11 |
+
DATA_DIR = "/data" if os.path.isdir("/data") and os.access("/data", os.W_OK) else \
|
| 12 |
+
os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "runtime")
|
| 13 |
+
os.makedirs(DATA_DIR, exist_ok=True)
|
| 14 |
+
DB_PATH = os.path.join(DATA_DIR, "nanodex.db")
|
| 15 |
+
RUNS_DIR = os.path.join(DATA_DIR, "runs")
|
| 16 |
+
os.makedirs(RUNS_DIR, exist_ok=True)
|
| 17 |
+
|
| 18 |
+
_lock = threading.RLock()
|
| 19 |
+
|
| 20 |
+
SCHEMA = """
|
| 21 |
+
CREATE TABLE IF NOT EXISTS jobs (
|
| 22 |
+
id TEXT PRIMARY KEY,
|
| 23 |
+
username TEXT NOT NULL,
|
| 24 |
+
display_name TEXT,
|
| 25 |
+
avatar_url TEXT,
|
| 26 |
+
model_name TEXT NOT NULL,
|
| 27 |
+
tier TEXT NOT NULL,
|
| 28 |
+
n_params INTEGER NOT NULL,
|
| 29 |
+
target_tokens INTEGER NOT NULL,
|
| 30 |
+
status TEXT NOT NULL, -- queued|running|done|failed|cancelled
|
| 31 |
+
gpu_index INTEGER,
|
| 32 |
+
created_at REAL NOT NULL,
|
| 33 |
+
started_at REAL,
|
| 34 |
+
finished_at REAL,
|
| 35 |
+
tokens_seen INTEGER DEFAULT 0,
|
| 36 |
+
step INTEGER DEFAULT 0,
|
| 37 |
+
total_steps INTEGER DEFAULT 0,
|
| 38 |
+
loss REAL,
|
| 39 |
+
best_loss REAL,
|
| 40 |
+
lr REAL,
|
| 41 |
+
tok_per_s REAL,
|
| 42 |
+
eta_s REAL,
|
| 43 |
+
error TEXT,
|
| 44 |
+
pushed_repo TEXT,
|
| 45 |
+
seed INTEGER DEFAULT 1337,
|
| 46 |
+
live_step INTEGER,
|
| 47 |
+
live_at REAL
|
| 48 |
+
);
|
| 49 |
+
CREATE INDEX IF NOT EXISTS idx_jobs_status ON jobs(status, created_at);
|
| 50 |
+
CREATE INDEX IF NOT EXISTS idx_jobs_user ON jobs(username, created_at DESC);
|
| 51 |
+
|
| 52 |
+
CREATE TABLE IF NOT EXISTS metrics (
|
| 53 |
+
job_id TEXT NOT NULL,
|
| 54 |
+
step INTEGER NOT NULL,
|
| 55 |
+
tokens INTEGER NOT NULL,
|
| 56 |
+
loss REAL NOT NULL,
|
| 57 |
+
lr REAL,
|
| 58 |
+
t REAL NOT NULL,
|
| 59 |
+
PRIMARY KEY (job_id, step)
|
| 60 |
+
);
|
| 61 |
+
|
| 62 |
+
CREATE TABLE IF NOT EXISTS logs (
|
| 63 |
+
job_id TEXT NOT NULL,
|
| 64 |
+
t REAL NOT NULL,
|
| 65 |
+
line TEXT NOT NULL
|
| 66 |
+
);
|
| 67 |
+
CREATE INDEX IF NOT EXISTS idx_logs_job ON logs(job_id, t);
|
| 68 |
+
"""
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
def _conn():
|
| 72 |
+
c = sqlite3.connect(DB_PATH, timeout=10, check_same_thread=False)
|
| 73 |
+
c.row_factory = sqlite3.Row
|
| 74 |
+
c.execute("PRAGMA journal_mode=WAL")
|
| 75 |
+
c.execute("PRAGMA busy_timeout=10000")
|
| 76 |
+
return c
|
| 77 |
+
|
| 78 |
+
|
| 79 |
+
# Columns added after the first release; applied to existing databases on boot.
|
| 80 |
+
MIGRATIONS = [
|
| 81 |
+
"ALTER TABLE jobs ADD COLUMN live_step INTEGER",
|
| 82 |
+
"ALTER TABLE jobs ADD COLUMN live_at REAL",
|
| 83 |
+
]
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
def init():
|
| 87 |
+
with _lock, _conn() as c:
|
| 88 |
+
c.executescript(SCHEMA)
|
| 89 |
+
for stmt in MIGRATIONS:
|
| 90 |
+
try:
|
| 91 |
+
c.execute(stmt)
|
| 92 |
+
except sqlite3.OperationalError:
|
| 93 |
+
pass # already present
|
| 94 |
+
|
| 95 |
+
# Any job left 'running' belongs to a container that is gone. This is a
|
| 96 |
+
# write, so it can contend with another process holding the database (a
|
| 97 |
+
# rolling deploy briefly runs two containers against the same volume) —
|
| 98 |
+
# never let that stall the boot.
|
| 99 |
+
try:
|
| 100 |
+
with _lock, _conn() as c:
|
| 101 |
+
c.execute(
|
| 102 |
+
"UPDATE jobs SET status='failed', "
|
| 103 |
+
"error='interrupted (the Space restarted)', finished_at=? "
|
| 104 |
+
"WHERE status='running'",
|
| 105 |
+
(time.time(),),
|
| 106 |
+
)
|
| 107 |
+
except sqlite3.Error as exc:
|
| 108 |
+
print(f"[db] could not reap orphaned runs: {exc!r}", flush=True)
|
| 109 |
+
|
| 110 |
+
|
| 111 |
+
def create_job(username, display_name, avatar_url, model_name, tier,
|
| 112 |
+
n_params, target_tokens, total_steps, seed=1337):
|
| 113 |
+
job_id = uuid.uuid4().hex[:12]
|
| 114 |
+
with _lock, _conn() as c:
|
| 115 |
+
c.execute(
|
| 116 |
+
"INSERT INTO jobs (id, username, display_name, avatar_url, model_name, tier,"
|
| 117 |
+
" n_params, target_tokens, status, created_at, total_steps, seed)"
|
| 118 |
+
" VALUES (?,?,?,?,?,?,?,?, 'queued', ?, ?, ?)",
|
| 119 |
+
(job_id, username, display_name, avatar_url, model_name, tier,
|
| 120 |
+
n_params, target_tokens, time.time(), total_steps, seed),
|
| 121 |
+
)
|
| 122 |
+
return job_id
|
| 123 |
+
|
| 124 |
+
|
| 125 |
+
def update_job(job_id, **fields):
|
| 126 |
+
if not fields:
|
| 127 |
+
return
|
| 128 |
+
cols = ", ".join(f"{k}=?" for k in fields)
|
| 129 |
+
with _lock, _conn() as c:
|
| 130 |
+
c.execute(f"UPDATE jobs SET {cols} WHERE id=?", (*fields.values(), job_id))
|
| 131 |
+
|
| 132 |
+
|
| 133 |
+
def get_job(job_id):
|
| 134 |
+
with _lock, _conn() as c:
|
| 135 |
+
r = c.execute("SELECT * FROM jobs WHERE id=?", (job_id,)).fetchone()
|
| 136 |
+
return dict(r) if r else None
|
| 137 |
+
|
| 138 |
+
|
| 139 |
+
def claim_next_queued(gpu_index):
|
| 140 |
+
"""Atomically move the oldest queued job to running. Returns the job or None."""
|
| 141 |
+
with _lock, _conn() as c:
|
| 142 |
+
r = c.execute(
|
| 143 |
+
"SELECT * FROM jobs WHERE status='queued' ORDER BY created_at LIMIT 1"
|
| 144 |
+
).fetchone()
|
| 145 |
+
if not r:
|
| 146 |
+
return None
|
| 147 |
+
n = c.execute(
|
| 148 |
+
"UPDATE jobs SET status='running', gpu_index=?, started_at=? "
|
| 149 |
+
"WHERE id=? AND status='queued'",
|
| 150 |
+
(gpu_index, time.time(), r["id"]),
|
| 151 |
+
).rowcount
|
| 152 |
+
if n == 0:
|
| 153 |
+
return None
|
| 154 |
+
return dict(c.execute("SELECT * FROM jobs WHERE id=?", (r["id"],)).fetchone())
|
| 155 |
+
|
| 156 |
+
|
| 157 |
+
def list_jobs(status=None, username=None, limit=100):
|
| 158 |
+
q = "SELECT * FROM jobs"
|
| 159 |
+
where, args = [], []
|
| 160 |
+
if status:
|
| 161 |
+
where.append("status IN (%s)" % ",".join("?" * len(status)))
|
| 162 |
+
args += list(status)
|
| 163 |
+
if username:
|
| 164 |
+
where.append("username=?")
|
| 165 |
+
args.append(username)
|
| 166 |
+
if where:
|
| 167 |
+
q += " WHERE " + " AND ".join(where)
|
| 168 |
+
q += " ORDER BY created_at DESC LIMIT ?"
|
| 169 |
+
args.append(limit)
|
| 170 |
+
with _lock, _conn() as c:
|
| 171 |
+
return [dict(r) for r in c.execute(q, args).fetchall()]
|
| 172 |
+
|
| 173 |
+
|
| 174 |
+
def queue_position(job_id):
|
| 175 |
+
with _lock, _conn() as c:
|
| 176 |
+
r = c.execute("SELECT created_at FROM jobs WHERE id=?", (job_id,)).fetchone()
|
| 177 |
+
if not r:
|
| 178 |
+
return None
|
| 179 |
+
return c.execute(
|
| 180 |
+
"SELECT COUNT(*) FROM jobs WHERE status='queued' AND created_at < ?",
|
| 181 |
+
(r["created_at"],),
|
| 182 |
+
).fetchone()[0] + 1
|
| 183 |
+
|
| 184 |
+
|
| 185 |
+
def user_active_count(username):
|
| 186 |
+
with _lock, _conn() as c:
|
| 187 |
+
return c.execute(
|
| 188 |
+
"SELECT COUNT(*) FROM jobs WHERE username=? AND status IN ('queued','running')",
|
| 189 |
+
(username,),
|
| 190 |
+
).fetchone()[0]
|
| 191 |
+
|
| 192 |
+
|
| 193 |
+
def add_metric(job_id, step, tokens, loss, lr, t=None):
|
| 194 |
+
with _lock, _conn() as c:
|
| 195 |
+
c.execute(
|
| 196 |
+
"INSERT OR REPLACE INTO metrics (job_id, step, tokens, loss, lr, t) VALUES (?,?,?,?,?,?)",
|
| 197 |
+
(job_id, step, tokens, loss, lr, t or time.time()),
|
| 198 |
+
)
|
| 199 |
+
|
| 200 |
+
|
| 201 |
+
def get_metrics(job_id, limit=3000):
|
| 202 |
+
with _lock, _conn() as c:
|
| 203 |
+
return [dict(r) for r in c.execute(
|
| 204 |
+
"SELECT step, tokens, loss, lr FROM metrics WHERE job_id=? ORDER BY step LIMIT ?",
|
| 205 |
+
(job_id, limit),
|
| 206 |
+
).fetchall()]
|
| 207 |
+
|
| 208 |
+
|
| 209 |
+
def add_log(job_id, line):
|
| 210 |
+
with _lock, _conn() as c:
|
| 211 |
+
c.execute("INSERT INTO logs (job_id, t, line) VALUES (?,?,?)",
|
| 212 |
+
(job_id, time.time(), line))
|
| 213 |
+
|
| 214 |
+
|
| 215 |
+
def get_logs(job_id, limit=300):
|
| 216 |
+
with _lock, _conn() as c:
|
| 217 |
+
rows = c.execute(
|
| 218 |
+
"SELECT t, line FROM logs WHERE job_id=? ORDER BY t DESC LIMIT ?",
|
| 219 |
+
(job_id, limit),
|
| 220 |
+
).fetchall()
|
| 221 |
+
return [dict(r) for r in reversed(rows)]
|
| 222 |
+
|
| 223 |
+
|
| 224 |
+
# ---------------------------------------------------------- community ------
|
| 225 |
+
_SORTS = {
|
| 226 |
+
"recent": "COALESCE(finished_at, created_at) DESC",
|
| 227 |
+
"loss": "best_loss ASC",
|
| 228 |
+
"tokens": "tokens_seen DESC",
|
| 229 |
+
"params": "n_params DESC, tokens_seen DESC",
|
| 230 |
+
}
|
| 231 |
+
|
| 232 |
+
|
| 233 |
+
def list_public(tier=None, sort="recent", limit=60, offset=0, username=None):
|
| 234 |
+
"""Finished runs from everyone — the site's shared shelf of models."""
|
| 235 |
+
q = ("SELECT * FROM jobs WHERE status='done' AND tokens_seen > 0 "
|
| 236 |
+
"AND best_loss IS NOT NULL")
|
| 237 |
+
args = []
|
| 238 |
+
if tier:
|
| 239 |
+
q += " AND tier=?"
|
| 240 |
+
args.append(tier)
|
| 241 |
+
if username:
|
| 242 |
+
q += " AND username=?"
|
| 243 |
+
args.append(username)
|
| 244 |
+
q += f" ORDER BY {_SORTS.get(sort, _SORTS['recent'])} LIMIT ? OFFSET ?"
|
| 245 |
+
args += [limit, offset]
|
| 246 |
+
with _lock, _conn() as c:
|
| 247 |
+
return [dict(r) for r in c.execute(q, args).fetchall()]
|
| 248 |
+
|
| 249 |
+
|
| 250 |
+
def list_active_public(limit=40):
|
| 251 |
+
"""Everything training or queued right now, from everyone."""
|
| 252 |
+
with _lock, _conn() as c:
|
| 253 |
+
return [dict(r) for r in c.execute(
|
| 254 |
+
"SELECT * FROM jobs WHERE status IN ('running','queued') "
|
| 255 |
+
"ORDER BY CASE status WHEN 'running' THEN 0 ELSE 1 END, created_at "
|
| 256 |
+
"LIMIT ?", (limit,)).fetchall()]
|
| 257 |
+
|
| 258 |
+
|
| 259 |
+
def leaderboard(limit_per_tier=3):
|
| 260 |
+
"""Lowest loss per tier. Same tokenizer, data and context for everyone,
|
| 261 |
+
so the numbers are directly comparable."""
|
| 262 |
+
out = {}
|
| 263 |
+
with _lock, _conn() as c:
|
| 264 |
+
tiers = [r[0] for r in c.execute(
|
| 265 |
+
"SELECT DISTINCT tier FROM jobs WHERE status='done' "
|
| 266 |
+
"AND best_loss IS NOT NULL").fetchall()]
|
| 267 |
+
for t in tiers:
|
| 268 |
+
out[t] = [dict(r) for r in c.execute(
|
| 269 |
+
"SELECT * FROM jobs WHERE status='done' AND tier=? "
|
| 270 |
+
"AND best_loss IS NOT NULL ORDER BY best_loss ASC LIMIT ?",
|
| 271 |
+
(t, limit_per_tier)).fetchall()]
|
| 272 |
+
return out
|
| 273 |
+
|
| 274 |
+
|
| 275 |
+
def user_profile(username):
|
| 276 |
+
"""Public totals for one person, plus the avatar they last signed in with."""
|
| 277 |
+
with _lock, _conn() as c:
|
| 278 |
+
row = c.execute(
|
| 279 |
+
"SELECT COUNT(*) AS runs,"
|
| 280 |
+
" SUM(CASE WHEN status='done' THEN 1 ELSE 0 END) AS done,"
|
| 281 |
+
" SUM(CASE WHEN status IN ('queued','running') THEN 1 ELSE 0 END) AS active,"
|
| 282 |
+
" COALESCE(SUM(tokens_seen),0) AS tokens,"
|
| 283 |
+
" MIN(CASE WHEN status='done' THEN best_loss END) AS best_loss,"
|
| 284 |
+
" SUM(CASE WHEN pushed_repo IS NOT NULL THEN 1 ELSE 0 END) AS published,"
|
| 285 |
+
" MIN(created_at) AS first_seen, MAX(created_at) AS last_seen"
|
| 286 |
+
" FROM jobs WHERE username=?", (username,)).fetchone()
|
| 287 |
+
if not row or not row["runs"]:
|
| 288 |
+
return None
|
| 289 |
+
meta = c.execute(
|
| 290 |
+
"SELECT display_name, avatar_url FROM jobs WHERE username=? "
|
| 291 |
+
"ORDER BY created_at DESC LIMIT 1", (username,)).fetchone()
|
| 292 |
+
params = c.execute(
|
| 293 |
+
"SELECT COALESCE(SUM(n_params),0) FROM jobs WHERE username=? "
|
| 294 |
+
"AND status='done'", (username,)).fetchone()[0]
|
| 295 |
+
d = dict(row)
|
| 296 |
+
d.update(username=username, params=params,
|
| 297 |
+
display_name=(meta["display_name"] if meta else None) or username,
|
| 298 |
+
avatar_url=meta["avatar_url"] if meta else None)
|
| 299 |
+
return d
|
| 300 |
+
|
| 301 |
+
|
| 302 |
+
def top_trainers(limit=12):
|
| 303 |
+
with _lock, _conn() as c:
|
| 304 |
+
return [dict(r) for r in c.execute(
|
| 305 |
+
"SELECT username, display_name, avatar_url,"
|
| 306 |
+
" COUNT(*) AS runs,"
|
| 307 |
+
" SUM(CASE WHEN status='done' THEN 1 ELSE 0 END) AS done,"
|
| 308 |
+
" COALESCE(SUM(tokens_seen),0) AS tokens,"
|
| 309 |
+
" MIN(CASE WHEN status='done' THEN best_loss END) AS best_loss"
|
| 310 |
+
" FROM jobs GROUP BY username"
|
| 311 |
+
" ORDER BY done DESC, tokens DESC LIMIT ?", (limit,)).fetchall()]
|
| 312 |
+
|
| 313 |
+
|
| 314 |
+
# -------------------------------------------------- durable round-trip -----
|
| 315 |
+
def export_run(job_id):
|
| 316 |
+
"""Everything about one run, in one JSON-serialisable dict."""
|
| 317 |
+
j = get_job(job_id)
|
| 318 |
+
if not j:
|
| 319 |
+
return None
|
| 320 |
+
return {"job": j, "metrics": get_metrics(job_id, 5000),
|
| 321 |
+
"logs": get_logs(job_id, 400), "schema": 1}
|
| 322 |
+
|
| 323 |
+
|
| 324 |
+
def import_run(payload):
|
| 325 |
+
"""Insert a run exported earlier. Existing rows are left untouched."""
|
| 326 |
+
j = payload.get("job") or {}
|
| 327 |
+
jid = j.get("id")
|
| 328 |
+
if not jid:
|
| 329 |
+
return False
|
| 330 |
+
cols = [
|
| 331 |
+
"id", "username", "display_name", "avatar_url", "model_name", "tier",
|
| 332 |
+
"n_params", "target_tokens", "status", "gpu_index", "created_at",
|
| 333 |
+
"started_at", "finished_at", "tokens_seen", "step", "total_steps",
|
| 334 |
+
"loss", "best_loss", "lr", "tok_per_s", "eta_s", "error",
|
| 335 |
+
"pushed_repo", "seed", "live_step", "live_at",
|
| 336 |
+
]
|
| 337 |
+
vals = [j.get(c) for c in cols]
|
| 338 |
+
with _lock, _conn() as c:
|
| 339 |
+
if c.execute("SELECT 1 FROM jobs WHERE id=?", (jid,)).fetchone():
|
| 340 |
+
return False
|
| 341 |
+
c.execute(
|
| 342 |
+
f"INSERT INTO jobs ({','.join(cols)}) "
|
| 343 |
+
f"VALUES ({','.join('?' * len(cols))})", vals)
|
| 344 |
+
for m in payload.get("metrics", []):
|
| 345 |
+
c.execute("INSERT OR REPLACE INTO metrics "
|
| 346 |
+
"(job_id, step, tokens, loss, lr, t) VALUES (?,?,?,?,?,?)",
|
| 347 |
+
(jid, m.get("step"), m.get("tokens"), m.get("loss"),
|
| 348 |
+
m.get("lr"), j.get("finished_at") or time.time()))
|
| 349 |
+
for l in payload.get("logs", []):
|
| 350 |
+
c.execute("INSERT INTO logs (job_id, t, line) VALUES (?,?,?)",
|
| 351 |
+
(jid, l.get("t") or time.time(), l.get("line", "")))
|
| 352 |
+
return True
|
| 353 |
+
|
| 354 |
+
|
| 355 |
+
def known_job_ids():
|
| 356 |
+
with _lock, _conn() as c:
|
| 357 |
+
return {r[0] for r in c.execute("SELECT id FROM jobs").fetchall()}
|
| 358 |
+
|
| 359 |
+
|
| 360 |
+
def stats():
|
| 361 |
+
with _lock, _conn() as c:
|
| 362 |
+
row = c.execute(
|
| 363 |
+
"SELECT "
|
| 364 |
+
" (SELECT COUNT(*) FROM jobs WHERE status='queued') AS queued,"
|
| 365 |
+
" (SELECT COUNT(*) FROM jobs WHERE status='running') AS running,"
|
| 366 |
+
" (SELECT COUNT(*) FROM jobs WHERE status='done') AS done,"
|
| 367 |
+
" (SELECT COUNT(DISTINCT username) FROM jobs) AS users,"
|
| 368 |
+
" (SELECT COALESCE(SUM(tokens_seen),0) FROM jobs) AS tokens"
|
| 369 |
+
).fetchone()
|
| 370 |
+
return dict(row)
|
nanodex/estimate.py
ADDED
|
@@ -0,0 +1,118 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Throughput estimates, calibrated from this Space's own completed runs."""
|
| 2 |
+
|
| 3 |
+
import sqlite3
|
| 4 |
+
|
| 5 |
+
from . import db
|
| 6 |
+
from .config import TIERS, SEQ_LEN, VOCAB_SIZE
|
| 7 |
+
|
| 8 |
+
# Two tiers have now been measured on a single L4, and that is enough to fit
|
| 9 |
+
# the cost model instead of guessing at it:
|
| 10 |
+
#
|
| 11 |
+
# seconds/token = HEAD + body_flops / RATE
|
| 12 |
+
#
|
| 13 |
+
# HEAD is the tied lm_head matmul plus the softmax/cross-entropy over the
|
| 14 |
+
# 2,048-token vocabulary. It depends on the number of tokens, not on the model
|
| 15 |
+
# size — 40% of the 500k tier's step time, 21% of the 1M's. RATE is the
|
| 16 |
+
# effective throughput of the transformer stack itself.
|
| 17 |
+
#
|
| 18 |
+
# Solving the two measurements gives HEAD ~= 0.54 us/token and
|
| 19 |
+
# RATE ~= 3.3 TFLOP/s. The larger tiers get a width correction because wider
|
| 20 |
+
# matmuls use the GPU better; that correction is the least certain part here,
|
| 21 |
+
# so the 5M and 8M numbers carry real uncertainty until runs of those sizes
|
| 22 |
+
# finish and replace the prior outright.
|
| 23 |
+
MEASURED_ANCHORS = { # tier -> tokens/s on one L4
|
| 24 |
+
"500k": 737_000.0,
|
| 25 |
+
"1m": 382_000.0,
|
| 26 |
+
}
|
| 27 |
+
_WIDTH_EXP = 0.6 # body efficiency vs hidden size
|
| 28 |
+
CPU_PENALTY = 60.0
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
def _body_flops_per_token(t):
|
| 32 |
+
"""FLOPs excluding the lm_head: the transformer stack and attention."""
|
| 33 |
+
non_emb = t.param_count() - VOCAB_SIZE * t.hidden_size
|
| 34 |
+
return 6 * non_emb + 6 * t.num_hidden_layers * SEQ_LEN * t.hidden_size
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def _fit():
|
| 38 |
+
"""Solve HEAD and RATE from the measured anchors."""
|
| 39 |
+
keys = [k for k in MEASURED_ANCHORS if k in TIERS]
|
| 40 |
+
if len(keys) < 2:
|
| 41 |
+
return None
|
| 42 |
+
keys.sort(key=lambda k: _body_flops_per_token(TIERS[k]))
|
| 43 |
+
lo, hi = keys[0], keys[-1]
|
| 44 |
+
f_lo, f_hi = (_body_flops_per_token(TIERS[k]) for k in (lo, hi))
|
| 45 |
+
t_lo, t_hi = (1.0 / MEASURED_ANCHORS[k] for k in (lo, hi))
|
| 46 |
+
if f_hi <= f_lo or t_hi <= t_lo:
|
| 47 |
+
return None
|
| 48 |
+
rate = (f_hi - f_lo) / (t_hi - t_lo)
|
| 49 |
+
head = t_lo - f_lo / rate
|
| 50 |
+
if head <= 0 or rate <= 0:
|
| 51 |
+
return None
|
| 52 |
+
return head, rate, TIERS[hi].hidden_size
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
_FIT = _fit()
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
def _prior(tier_key):
|
| 59 |
+
if tier_key in MEASURED_ANCHORS:
|
| 60 |
+
return MEASURED_ANCHORS[tier_key]
|
| 61 |
+
t = TIERS[tier_key]
|
| 62 |
+
if not _FIT:
|
| 63 |
+
return 300_000.0
|
| 64 |
+
head, rate, ref_h = _FIT
|
| 65 |
+
rate = rate * (t.hidden_size / ref_h) ** _WIDTH_EXP
|
| 66 |
+
return 1.0 / (head + _body_flops_per_token(t) / rate)
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
PRIOR_TOK_S = {k: _prior(k) for k in TIERS}
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
def measured_tok_s(tier_key):
|
| 73 |
+
try:
|
| 74 |
+
with db._lock, db._conn() as c:
|
| 75 |
+
rows = c.execute(
|
| 76 |
+
"SELECT tok_per_s FROM jobs WHERE tier=? AND status='done' "
|
| 77 |
+
"AND tok_per_s > 0 ORDER BY finished_at DESC LIMIT 9",
|
| 78 |
+
(tier_key,),
|
| 79 |
+
).fetchall()
|
| 80 |
+
vals = sorted(r[0] for r in rows)
|
| 81 |
+
if len(vals) >= 2:
|
| 82 |
+
return vals[len(vals) // 2]
|
| 83 |
+
except sqlite3.Error:
|
| 84 |
+
pass
|
| 85 |
+
return None
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
def tok_s_for(tier_key, on_gpu=True):
|
| 89 |
+
m = measured_tok_s(tier_key)
|
| 90 |
+
if m:
|
| 91 |
+
return m
|
| 92 |
+
base = PRIOR_TOK_S[tier_key]
|
| 93 |
+
return base if on_gpu else base / CPU_PENALTY
|
| 94 |
+
|
| 95 |
+
|
| 96 |
+
def eta_seconds(tier_key, target_tokens, on_gpu=True):
|
| 97 |
+
return target_tokens / max(tok_s_for(tier_key, on_gpu), 1.0)
|
| 98 |
+
|
| 99 |
+
|
| 100 |
+
def human_time(sec):
|
| 101 |
+
sec = max(sec, 0)
|
| 102 |
+
if sec < 90:
|
| 103 |
+
return f"{sec:.0f}s"
|
| 104 |
+
if sec < 5400:
|
| 105 |
+
return f"{sec / 60:.0f} min"
|
| 106 |
+
return f"{sec / 3600:.1f} h"
|
| 107 |
+
|
| 108 |
+
|
| 109 |
+
def summary(tier_key, target_tokens, on_gpu=True):
|
| 110 |
+
tier = TIERS[tier_key]
|
| 111 |
+
n = tier.param_count()
|
| 112 |
+
steps = max(1, target_tokens // tier.batch_tokens)
|
| 113 |
+
eta = eta_seconds(tier_key, target_tokens, on_gpu)
|
| 114 |
+
ratio = target_tokens / n
|
| 115 |
+
return {
|
| 116 |
+
"n_params": n, "steps": steps, "eta_s": eta, "ratio": ratio,
|
| 117 |
+
"calibrated": measured_tok_s(tier_key) is not None,
|
| 118 |
+
}
|
nanodex/hub.py
ADDED
|
@@ -0,0 +1,111 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Push a finished HyperDex run to the user's own Hugging Face account."""
|
| 2 |
+
|
| 3 |
+
import json
|
| 4 |
+
import os
|
| 5 |
+
|
| 6 |
+
from huggingface_hub import HfApi
|
| 7 |
+
|
| 8 |
+
from .config import TIERS
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
def model_card(job, run_meta, repo_id):
|
| 12 |
+
tier = TIERS[job["tier"]]
|
| 13 |
+
loss = run_meta.get("final_loss")
|
| 14 |
+
import math
|
| 15 |
+
ppl = math.exp(min(loss, 20)) if loss else None
|
| 16 |
+
return f"""---
|
| 17 |
+
license: odc-by
|
| 18 |
+
datasets:
|
| 19 |
+
- HuggingFaceFW/fineweb-edu
|
| 20 |
+
language:
|
| 21 |
+
- en
|
| 22 |
+
library_name: transformers
|
| 23 |
+
pipeline_tag: text-generation
|
| 24 |
+
tags:
|
| 25 |
+
- nanodex
|
| 26 |
+
- tiny-lm
|
| 27 |
+
- pretrained-from-scratch
|
| 28 |
+
---
|
| 29 |
+
|
| 30 |
+
# {repo_id.split('/')[-1]}
|
| 31 |
+
|
| 32 |
+
A **{run_meta['n_params']:,}-parameter** decoder-only language model pre-trained
|
| 33 |
+
**from scratch** on [fineweb-edu](https://huggingface.co/datasets/HuggingFaceFW/fineweb-edu),
|
| 34 |
+
using the [HyperDex Trainer](https://hugging-science-hyperdex-trainer.hf.space/) Space.
|
| 35 |
+
|
| 36 |
+
## Architecture
|
| 37 |
+
|
| 38 |
+
A standard `LlamaForCausalLM` decoder-only transformer — SiLU MLP, RMSNorm,
|
| 39 |
+
rotary position embeddings, grouped-query attention, tied embeddings, no biases —
|
| 40 |
+
scaled down in width and depth to fit the parameter budget.
|
| 41 |
+
|
| 42 |
+
| | |
|
| 43 |
+
|---|---|
|
| 44 |
+
| Parameters | {run_meta['n_params']:,} |
|
| 45 |
+
| Hidden size | {tier.hidden_size} |
|
| 46 |
+
| Layers | {tier.num_hidden_layers} |
|
| 47 |
+
| Attention heads | {tier.num_attention_heads} (KV: {tier.num_key_value_heads}) |
|
| 48 |
+
| FFN size | {tier.intermediate_size} |
|
| 49 |
+
| Context length | {run_meta['seq_len']} |
|
| 50 |
+
| Vocab | 2,048 (custom BPE trained on fineweb-edu) |
|
| 51 |
+
|
| 52 |
+
## Training
|
| 53 |
+
|
| 54 |
+
| | |
|
| 55 |
+
|---|---|
|
| 56 |
+
| Tokens seen | {run_meta['tokens_seen']:,} |
|
| 57 |
+
| Steps | {run_meta['steps']:,} |
|
| 58 |
+
| Tokens / step | {run_meta['batch_tokens']:,} |
|
| 59 |
+
| Optimizer | {run_meta['optimizer']} |
|
| 60 |
+
| LR schedule | {run_meta['schedule']} (peak {run_meta['peak_lr']:.0e}) |
|
| 61 |
+
| Final loss | {loss:.4f}{f" (ppl {ppl:.1f})" if ppl else ""} |
|
| 62 |
+
| Wall time | {run_meta['wall_time_s'] / 60:.1f} min |
|
| 63 |
+
| Trained by | [@{job['username']}](https://huggingface.co/{job['username']}) |
|
| 64 |
+
|
| 65 |
+
## Usage
|
| 66 |
+
|
| 67 |
+
```python
|
| 68 |
+
from transformers import AutoModelForCausalLM, AutoTokenizer
|
| 69 |
+
|
| 70 |
+
tok = AutoTokenizer.from_pretrained("{repo_id}")
|
| 71 |
+
model = AutoModelForCausalLM.from_pretrained("{repo_id}")
|
| 72 |
+
|
| 73 |
+
ids = tok("The mitochondria is", return_tensors="pt").input_ids
|
| 74 |
+
print(tok.decode(model.generate(ids, max_new_tokens=60, do_sample=True,
|
| 75 |
+
temperature=0.8, top_k=50)[0]))
|
| 76 |
+
```
|
| 77 |
+
|
| 78 |
+
## Caveats
|
| 79 |
+
|
| 80 |
+
This is a **small-scale research artifact**. At this parameter count and token
|
| 81 |
+
budget the model learns word shapes, common collocations and a little syntax —
|
| 82 |
+
it is not a useful assistant and its output is not factual. It exists to make
|
| 83 |
+
"pre-train a transformer from scratch" something you can actually watch happen.
|
| 84 |
+
"""
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
def push_run(out_dir, job, token, repo_name, private=False):
|
| 88 |
+
api = HfApi(token=token)
|
| 89 |
+
who = api.whoami()
|
| 90 |
+
owner = who["name"]
|
| 91 |
+
repo_id = f"{owner}/{repo_name}"
|
| 92 |
+
|
| 93 |
+
api.create_repo(repo_id, repo_type="model", private=private, exist_ok=True)
|
| 94 |
+
|
| 95 |
+
with open(os.path.join(out_dir, "training_run.json")) as f:
|
| 96 |
+
run_meta = json.load(f)
|
| 97 |
+
|
| 98 |
+
with open(os.path.join(out_dir, "README.md"), "w") as f:
|
| 99 |
+
f.write(model_card(job, run_meta, repo_id))
|
| 100 |
+
|
| 101 |
+
api.upload_folder(
|
| 102 |
+
folder_path=out_dir,
|
| 103 |
+
repo_id=repo_id,
|
| 104 |
+
repo_type="model",
|
| 105 |
+
# `live/` holds the mid-training snapshots that power the in-page
|
| 106 |
+
# playground; they have no business in the published model.
|
| 107 |
+
ignore_patterns=["live/*", "live", "*.tmp", "*.lock"],
|
| 108 |
+
commit_message=f"HyperDex {job['tier']} · {run_meta['tokens_seen']:,} fineweb-edu "
|
| 109 |
+
f"tokens · loss {run_meta['final_loss']:.4f}",
|
| 110 |
+
)
|
| 111 |
+
return repo_id
|
nanodex/store.py
ADDED
|
@@ -0,0 +1,259 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Durable storage on a Hugging Face bucket.
|
| 3 |
+
|
| 4 |
+
A Space's container filesystem dies on every rebuild, restart and garbage
|
| 5 |
+
collection, which would otherwise mean losing every model, every loss curve and
|
| 6 |
+
the whole 1.5B-token cache each time the code changes. This module mirrors the
|
| 7 |
+
three things worth keeping into a bucket:
|
| 8 |
+
|
| 9 |
+
cache/tokens.u16 + cache/tokens.meta the shared fineweb-edu token cache
|
| 10 |
+
runs/<job_id>/meta.json the job row, its metrics and logs
|
| 11 |
+
runs/<job_id>/<weights, tokenizer, …> the finished checkpoint
|
| 12 |
+
|
| 13 |
+
The design is deliberately append-only and keyed by job id: a finished run is
|
| 14 |
+
written once and never mutated. That removes every concurrency problem a rolling
|
| 15 |
+
deploy would otherwise create, because two containers never contend for the same
|
| 16 |
+
object — unlike syncing one shared SQLite file, where the last writer would
|
| 17 |
+
silently clobber the other.
|
| 18 |
+
|
| 19 |
+
On boot the local database is rebuilt from the run metadata in the bucket.
|
| 20 |
+
Checkpoints are pulled on demand, the first time someone opens or publishes a
|
| 21 |
+
model, so startup stays fast.
|
| 22 |
+
|
| 23 |
+
Runs that were in flight when a container died are NOT recoverable — the
|
| 24 |
+
optimizer state lives only in that process. They come back marked interrupted.
|
| 25 |
+
"""
|
| 26 |
+
|
| 27 |
+
import json
|
| 28 |
+
import os
|
| 29 |
+
import shutil
|
| 30 |
+
import threading
|
| 31 |
+
import time
|
| 32 |
+
|
| 33 |
+
BUCKET_ID = os.environ.get("NANODEX_BUCKET", "hugging-science/nanodex-store")
|
| 34 |
+
CACHE_PREFIX = "cache/"
|
| 35 |
+
RUNS_PREFIX = "runs/"
|
| 36 |
+
|
| 37 |
+
# Files worth keeping from a finished run directory.
|
| 38 |
+
CHECKPOINT_FILES = (
|
| 39 |
+
"config.json",
|
| 40 |
+
"generation_config.json",
|
| 41 |
+
"model.safetensors",
|
| 42 |
+
"tokenizer.json",
|
| 43 |
+
"tokenizer_config.json",
|
| 44 |
+
"special_tokens_map.json",
|
| 45 |
+
"training_run.json",
|
| 46 |
+
)
|
| 47 |
+
|
| 48 |
+
_lock = threading.Lock()
|
| 49 |
+
_state = {"enabled": None, "error": None, "restored": 0, "last_error_at": 0}
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def _api():
|
| 53 |
+
from huggingface_hub import HfApi
|
| 54 |
+
tok = os.environ.get("HF_TOKEN") or os.environ.get("NANODEX_HF_TOKEN")
|
| 55 |
+
return HfApi(token=tok) if tok else HfApi()
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
def state():
|
| 59 |
+
return dict(_state)
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
def available(log=print):
|
| 63 |
+
"""True when the bucket is reachable and writable by this Space."""
|
| 64 |
+
if _state["enabled"] is not None:
|
| 65 |
+
return _state["enabled"]
|
| 66 |
+
with _lock:
|
| 67 |
+
if _state["enabled"] is not None:
|
| 68 |
+
return _state["enabled"]
|
| 69 |
+
try:
|
| 70 |
+
_api().bucket_info(BUCKET_ID)
|
| 71 |
+
_state["enabled"] = True
|
| 72 |
+
log(f"[store] bucket ready: {BUCKET_ID}")
|
| 73 |
+
except Exception as exc:
|
| 74 |
+
_state.update(enabled=False, error=repr(exc))
|
| 75 |
+
log(f"[store] bucket unavailable ({exc!r}) — "
|
| 76 |
+
"running with container-local storage only")
|
| 77 |
+
return _state["enabled"]
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
def _note_error(what, exc, log=print):
|
| 81 |
+
_state.update(error=f"{what}: {exc!r}", last_error_at=time.time())
|
| 82 |
+
log(f"[store] {what} failed: {exc!r}")
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
# --------------------------------------------------------------- uploads ----
|
| 86 |
+
def put_files(pairs, log=print):
|
| 87 |
+
"""pairs: [(local_path, remote_path)] — uploaded in one batch."""
|
| 88 |
+
if not pairs or not available(log):
|
| 89 |
+
return False
|
| 90 |
+
try:
|
| 91 |
+
_api().batch_bucket_files(
|
| 92 |
+
BUCKET_ID, add=[(str(l), r) for l, r in pairs])
|
| 93 |
+
return True
|
| 94 |
+
except Exception as exc:
|
| 95 |
+
_note_error(f"upload of {len(pairs)} file(s)", exc, log)
|
| 96 |
+
return False
|
| 97 |
+
|
| 98 |
+
|
| 99 |
+
def put_json(obj, remote_path, log=print):
|
| 100 |
+
if not available(log):
|
| 101 |
+
return False
|
| 102 |
+
try:
|
| 103 |
+
payload = json.dumps(obj, indent=2).encode()
|
| 104 |
+
_api().batch_bucket_files(BUCKET_ID, add=[(payload, remote_path)])
|
| 105 |
+
return True
|
| 106 |
+
except Exception as exc:
|
| 107 |
+
_note_error(f"upload of {remote_path}", exc, log)
|
| 108 |
+
return False
|
| 109 |
+
|
| 110 |
+
|
| 111 |
+
def save_run(job_id, run_dir, meta, log=print):
|
| 112 |
+
"""Persist one finished run: its metadata and its checkpoint files."""
|
| 113 |
+
if not available(log):
|
| 114 |
+
return False
|
| 115 |
+
ok = put_json(meta, f"{RUNS_PREFIX}{job_id}/meta.json", log)
|
| 116 |
+
pairs = []
|
| 117 |
+
for fn in CHECKPOINT_FILES:
|
| 118 |
+
p = os.path.join(run_dir, fn)
|
| 119 |
+
if os.path.exists(p):
|
| 120 |
+
pairs.append((p, f"{RUNS_PREFIX}{job_id}/{fn}"))
|
| 121 |
+
if pairs:
|
| 122 |
+
ok = put_files(pairs, log) and ok
|
| 123 |
+
log(f"[store] saved run {job_id} ({len(pairs)} files)")
|
| 124 |
+
return ok
|
| 125 |
+
|
| 126 |
+
|
| 127 |
+
def save_token_cache(tokens_path, meta_path, log=print):
|
| 128 |
+
"""Upload the shared token cache so the next container skips tokenizing."""
|
| 129 |
+
if not available(log) or not os.path.exists(tokens_path):
|
| 130 |
+
return False
|
| 131 |
+
log("[store] uploading token cache (this happens once)")
|
| 132 |
+
t0 = time.time()
|
| 133 |
+
ok = put_files([(tokens_path, CACHE_PREFIX + "tokens.u16"),
|
| 134 |
+
(meta_path, CACHE_PREFIX + "tokens.meta")], log)
|
| 135 |
+
if ok:
|
| 136 |
+
log(f"[store] token cache uploaded in {time.time() - t0:.0f}s")
|
| 137 |
+
return ok
|
| 138 |
+
|
| 139 |
+
|
| 140 |
+
# ------------------------------------------------------------- downloads ----
|
| 141 |
+
def _remote_size(path, log=print):
|
| 142 |
+
try:
|
| 143 |
+
return _api().get_bucket_file_metadata(BUCKET_ID, path).size
|
| 144 |
+
except Exception:
|
| 145 |
+
return None
|
| 146 |
+
|
| 147 |
+
|
| 148 |
+
def fetch_token_cache(tokens_path, meta_path, log=print):
|
| 149 |
+
"""
|
| 150 |
+
Pull the token cache if the bucket has one at least as complete as what is
|
| 151 |
+
on disk. Returns the number of tokens now available locally, or 0.
|
| 152 |
+
"""
|
| 153 |
+
if not available(log):
|
| 154 |
+
return 0
|
| 155 |
+
try:
|
| 156 |
+
raw = _api().download_bucket_files(
|
| 157 |
+
BUCKET_ID,
|
| 158 |
+
[(CACHE_PREFIX + "tokens.meta", meta_path + ".remote")],
|
| 159 |
+
raise_on_missing_files=True)
|
| 160 |
+
except Exception:
|
| 161 |
+
return 0 # nothing stored yet — normal first boot
|
| 162 |
+
try:
|
| 163 |
+
with open(meta_path + ".remote") as f:
|
| 164 |
+
remote_tokens = int(f.read().strip())
|
| 165 |
+
except Exception:
|
| 166 |
+
return 0
|
| 167 |
+
finally:
|
| 168 |
+
try:
|
| 169 |
+
os.unlink(meta_path + ".remote")
|
| 170 |
+
except OSError:
|
| 171 |
+
pass
|
| 172 |
+
|
| 173 |
+
local_tokens = 0
|
| 174 |
+
if os.path.exists(meta_path):
|
| 175 |
+
try:
|
| 176 |
+
with open(meta_path) as f:
|
| 177 |
+
local_tokens = int(f.read().strip())
|
| 178 |
+
except Exception:
|
| 179 |
+
local_tokens = 0
|
| 180 |
+
if remote_tokens <= local_tokens:
|
| 181 |
+
return local_tokens
|
| 182 |
+
|
| 183 |
+
log(f"[store] downloading cached {remote_tokens:,} tokens "
|
| 184 |
+
f"(beats re-tokenizing from scratch)")
|
| 185 |
+
t0 = time.time()
|
| 186 |
+
try:
|
| 187 |
+
_api().download_bucket_files(
|
| 188 |
+
BUCKET_ID, [(CACHE_PREFIX + "tokens.u16", tokens_path)],
|
| 189 |
+
raise_on_missing_files=True)
|
| 190 |
+
with open(meta_path, "w") as f:
|
| 191 |
+
f.write(str(remote_tokens))
|
| 192 |
+
log(f"[store] token cache restored in {time.time() - t0:.0f}s")
|
| 193 |
+
return remote_tokens
|
| 194 |
+
except Exception as exc:
|
| 195 |
+
_note_error("token cache download", exc, log)
|
| 196 |
+
return local_tokens
|
| 197 |
+
|
| 198 |
+
|
| 199 |
+
def list_runs(log=print):
|
| 200 |
+
"""Job ids that have metadata stored in the bucket."""
|
| 201 |
+
if not available(log):
|
| 202 |
+
return []
|
| 203 |
+
try:
|
| 204 |
+
ids = set()
|
| 205 |
+
for entry in _api().list_bucket_tree(BUCKET_ID, prefix=RUNS_PREFIX,
|
| 206 |
+
recursive=True):
|
| 207 |
+
path = getattr(entry, "path", None) or getattr(entry, "name", "")
|
| 208 |
+
rest = str(path)[len(RUNS_PREFIX):] if str(path).startswith(RUNS_PREFIX) else ""
|
| 209 |
+
if "/" in rest:
|
| 210 |
+
ids.add(rest.split("/", 1)[0])
|
| 211 |
+
return sorted(ids)
|
| 212 |
+
except Exception as exc:
|
| 213 |
+
_note_error("listing runs", exc, log)
|
| 214 |
+
return []
|
| 215 |
+
|
| 216 |
+
|
| 217 |
+
def fetch_run_meta(job_id, log=print):
|
| 218 |
+
if not available(log):
|
| 219 |
+
return None
|
| 220 |
+
tmp = f"/tmp/nanodex-meta-{job_id}.json"
|
| 221 |
+
try:
|
| 222 |
+
_api().download_bucket_files(
|
| 223 |
+
BUCKET_ID, [(f"{RUNS_PREFIX}{job_id}/meta.json", tmp)],
|
| 224 |
+
raise_on_missing_files=True)
|
| 225 |
+
with open(tmp) as f:
|
| 226 |
+
return json.load(f)
|
| 227 |
+
except Exception:
|
| 228 |
+
return None
|
| 229 |
+
finally:
|
| 230 |
+
try:
|
| 231 |
+
os.unlink(tmp)
|
| 232 |
+
except OSError:
|
| 233 |
+
pass
|
| 234 |
+
|
| 235 |
+
|
| 236 |
+
def fetch_checkpoint(job_id, run_dir, log=print):
|
| 237 |
+
"""
|
| 238 |
+
Restore a finished run's weights into run_dir. Called lazily, the first
|
| 239 |
+
time somebody opens the playground or publishes after a restart.
|
| 240 |
+
"""
|
| 241 |
+
if not available(log):
|
| 242 |
+
return False
|
| 243 |
+
if os.path.exists(os.path.join(run_dir, "model.safetensors")):
|
| 244 |
+
return True
|
| 245 |
+
os.makedirs(run_dir, exist_ok=True)
|
| 246 |
+
wanted = [(f"{RUNS_PREFIX}{job_id}/{fn}", os.path.join(run_dir, fn))
|
| 247 |
+
for fn in CHECKPOINT_FILES]
|
| 248 |
+
try:
|
| 249 |
+
_api().download_bucket_files(BUCKET_ID, wanted,
|
| 250 |
+
raise_on_missing_files=False)
|
| 251 |
+
except Exception as exc:
|
| 252 |
+
_note_error(f"checkpoint download for {job_id}", exc, log)
|
| 253 |
+
return False
|
| 254 |
+
ok = os.path.exists(os.path.join(run_dir, "model.safetensors"))
|
| 255 |
+
if ok:
|
| 256 |
+
log(f"[store] restored checkpoint {job_id}")
|
| 257 |
+
else:
|
| 258 |
+
shutil.rmtree(run_dir, ignore_errors=True)
|
| 259 |
+
return ok
|
nanodex/trainer.py
ADDED
|
@@ -0,0 +1,254 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Pre-training loop for a single NanoDex run."""
|
| 2 |
+
|
| 3 |
+
import json
|
| 4 |
+
import math
|
| 5 |
+
import os
|
| 6 |
+
import shutil
|
| 7 |
+
import time
|
| 8 |
+
|
| 9 |
+
from . import db, store
|
| 10 |
+
from .config import TIERS, SEQ_LEN, VOCAB_SIZE
|
| 11 |
+
|
| 12 |
+
# Metric cadence adapts to run length so short runs still get a real curve.
|
| 13 |
+
def _cadence(total):
|
| 14 |
+
return max(1, min(25, total // 200))
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
def total_steps_for(tier_key, target_tokens):
|
| 18 |
+
tier = TIERS[tier_key]
|
| 19 |
+
return max(1, int(target_tokens // tier.batch_tokens))
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def _lr_at(step, total, base_lr, warmup_frac=0.02, min_frac=0.1):
|
| 23 |
+
warmup = max(1, int(total * warmup_frac))
|
| 24 |
+
if step < warmup:
|
| 25 |
+
return base_lr * (step + 1) / warmup
|
| 26 |
+
prog = (step - warmup) / max(1, total - warmup)
|
| 27 |
+
prog = min(max(prog, 0.0), 1.0)
|
| 28 |
+
return base_lr * (min_frac + (1 - min_frac) * 0.5 * (1 + math.cos(math.pi * prog)))
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
LIVE_SNAPSHOTS = 40 # snapshots spread over a run
|
| 32 |
+
LIVE_MIN_SECONDS = 25 # ...but never more often than this
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
def live_dir(job_id):
|
| 36 |
+
return os.path.join(db.RUNS_DIR, job_id, "live")
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
def _write_live_checkpoint(model, job_id, step, tokenizer_dir, first=False):
|
| 40 |
+
"""
|
| 41 |
+
Dump the current weights so the run page can generate text from the model
|
| 42 |
+
mid-training. Written to a temp file and renamed, so a reader never sees a
|
| 43 |
+
half-written checkpoint.
|
| 44 |
+
"""
|
| 45 |
+
import torch
|
| 46 |
+
from safetensors.torch import save_file
|
| 47 |
+
|
| 48 |
+
d = live_dir(job_id)
|
| 49 |
+
os.makedirs(d, exist_ok=True)
|
| 50 |
+
if first:
|
| 51 |
+
cfg = model.config.to_dict()
|
| 52 |
+
cfg["use_cache"] = True
|
| 53 |
+
cfg["torch_dtype"] = "float32"
|
| 54 |
+
with open(os.path.join(d, "config.json"), "w") as f:
|
| 55 |
+
json.dump(cfg, f, indent=2)
|
| 56 |
+
with open(os.path.join(d, "generation_config.json"), "w") as f:
|
| 57 |
+
json.dump({"bos_token_id": 0, "eos_token_id": 0,
|
| 58 |
+
"pad_token_id": 1}, f, indent=2)
|
| 59 |
+
for fn in ("tokenizer.json", "tokenizer_config.json",
|
| 60 |
+
"special_tokens_map.json"):
|
| 61 |
+
src = os.path.join(tokenizer_dir, fn)
|
| 62 |
+
if os.path.exists(src):
|
| 63 |
+
shutil.copy(src, os.path.join(d, fn))
|
| 64 |
+
|
| 65 |
+
sd = {k: v.detach().to("cpu", torch.float32).contiguous()
|
| 66 |
+
for k, v in model.state_dict().items()
|
| 67 |
+
if not k.endswith("lm_head.weight")} # tied to the embedding
|
| 68 |
+
tmp = os.path.join(d, "model.safetensors.tmp")
|
| 69 |
+
save_file(sd, tmp, metadata={"format": "pt", "step": str(step)})
|
| 70 |
+
os.replace(tmp, os.path.join(d, "model.safetensors"))
|
| 71 |
+
db.update_job(job_id, live_step=step, live_at=time.time())
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
def build_model(tier_key, vocab_size=VOCAB_SIZE):
|
| 75 |
+
import torch
|
| 76 |
+
try:
|
| 77 |
+
# transformers >= 5 dropped the top-level re-exports
|
| 78 |
+
from transformers.models.llama import LlamaConfig, LlamaForCausalLM
|
| 79 |
+
except ImportError:
|
| 80 |
+
from transformers import LlamaConfig, LlamaForCausalLM
|
| 81 |
+
|
| 82 |
+
tier = TIERS[tier_key]
|
| 83 |
+
cfg = LlamaConfig(**tier.to_hf_config_dict(vocab_size))
|
| 84 |
+
cfg.use_cache = False
|
| 85 |
+
model = LlamaForCausalLM(cfg)
|
| 86 |
+
return model
|
| 87 |
+
|
| 88 |
+
|
| 89 |
+
def run_training(job, device, tokenizer_dir, cancel_flag, log):
|
| 90 |
+
"""
|
| 91 |
+
Execute one job to completion. `cancel_flag` is a callable returning True
|
| 92 |
+
when the user asked to stop. Returns the output directory.
|
| 93 |
+
"""
|
| 94 |
+
import torch
|
| 95 |
+
from .data import TokenStream
|
| 96 |
+
|
| 97 |
+
job_id = job["id"]
|
| 98 |
+
tier = TIERS[job["tier"]]
|
| 99 |
+
target_tokens = int(job["target_tokens"])
|
| 100 |
+
total = total_steps_for(job["tier"], target_tokens)
|
| 101 |
+
seq_len = SEQ_LEN
|
| 102 |
+
|
| 103 |
+
use_cuda = device.startswith("cuda")
|
| 104 |
+
torch.manual_seed(int(job.get("seed") or 1337))
|
| 105 |
+
|
| 106 |
+
model = build_model(job["tier"]).to(device)
|
| 107 |
+
n_params = sum(p.numel() for p in model.parameters())
|
| 108 |
+
log(f"model: {tier.label} · {n_params:,} parameters · device={device}")
|
| 109 |
+
|
| 110 |
+
if use_cuda:
|
| 111 |
+
torch.backends.cuda.matmul.allow_tf32 = True
|
| 112 |
+
torch.backends.cudnn.allow_tf32 = True
|
| 113 |
+
|
| 114 |
+
micro = tier.micro_batch if use_cuda else max(4, tier.micro_batch // 32)
|
| 115 |
+
tokens_per_micro = micro * seq_len
|
| 116 |
+
accum = max(1, tier.batch_tokens // tokens_per_micro)
|
| 117 |
+
log(f"batch: {micro} seq x {seq_len} tok x {accum} accum "
|
| 118 |
+
f"= {micro * seq_len * accum:,} tokens/step · {total:,} steps")
|
| 119 |
+
|
| 120 |
+
opt = torch.optim.AdamW(
|
| 121 |
+
model.parameters(), lr=tier.lr, betas=(0.9, 0.95),
|
| 122 |
+
weight_decay=0.1, eps=1e-8, fused=use_cuda,
|
| 123 |
+
)
|
| 124 |
+
stream = TokenStream(seq_len, micro, seed=int(job.get("seed") or 1337))
|
| 125 |
+
amp_dtype = torch.bfloat16 if use_cuda else torch.float32
|
| 126 |
+
|
| 127 |
+
out_dir = os.path.join(db.RUNS_DIR, job_id)
|
| 128 |
+
os.makedirs(out_dir, exist_ok=True)
|
| 129 |
+
|
| 130 |
+
every = _cadence(total)
|
| 131 |
+
live_every = max(1, total // LIVE_SNAPSHOTS)
|
| 132 |
+
last_live = 0.0
|
| 133 |
+
live_started = False
|
| 134 |
+
model.train()
|
| 135 |
+
tokens_seen = 0
|
| 136 |
+
best = float("inf")
|
| 137 |
+
t_start = time.time()
|
| 138 |
+
t_win = time.time()
|
| 139 |
+
tok_win = 0
|
| 140 |
+
|
| 141 |
+
for step in range(total):
|
| 142 |
+
if cancel_flag():
|
| 143 |
+
log("cancelled by the user")
|
| 144 |
+
db.update_job(job_id, status="cancelled", finished_at=time.time())
|
| 145 |
+
return None
|
| 146 |
+
|
| 147 |
+
lr = _lr_at(step, total, tier.lr)
|
| 148 |
+
for g in opt.param_groups:
|
| 149 |
+
g["lr"] = lr
|
| 150 |
+
|
| 151 |
+
while True:
|
| 152 |
+
opt.zero_grad(set_to_none=True)
|
| 153 |
+
loss_acc = 0.0
|
| 154 |
+
step_tokens = 0
|
| 155 |
+
try:
|
| 156 |
+
for _ in range(accum):
|
| 157 |
+
x = stream.next_batch().to(device, non_blocking=True)
|
| 158 |
+
with torch.autocast(device_type="cuda" if use_cuda else "cpu",
|
| 159 |
+
dtype=amp_dtype, enabled=use_cuda):
|
| 160 |
+
# labels == input_ids: the model does the causal shift
|
| 161 |
+
# itself. Passing a pre-shifted target would train it to
|
| 162 |
+
# predict two tokens ahead of where generation reads.
|
| 163 |
+
out = model(input_ids=x, labels=x)
|
| 164 |
+
loss = out.loss / accum
|
| 165 |
+
loss.backward()
|
| 166 |
+
loss_acc += loss.item()
|
| 167 |
+
step_tokens += x.numel()
|
| 168 |
+
break
|
| 169 |
+
except torch.cuda.OutOfMemoryError:
|
| 170 |
+
if micro <= 4:
|
| 171 |
+
raise
|
| 172 |
+
micro = max(4, micro // 2)
|
| 173 |
+
accum = max(1, tier.batch_tokens // (micro * seq_len))
|
| 174 |
+
opt.zero_grad(set_to_none=True)
|
| 175 |
+
torch.cuda.empty_cache()
|
| 176 |
+
stream = TokenStream(seq_len, micro,
|
| 177 |
+
seed=int(job.get("seed") or 1337))
|
| 178 |
+
log(f"out of GPU memory — retrying with {micro} seq x {accum} accum")
|
| 179 |
+
|
| 180 |
+
tokens_seen += step_tokens
|
| 181 |
+
tok_win += step_tokens
|
| 182 |
+
|
| 183 |
+
torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
|
| 184 |
+
opt.step()
|
| 185 |
+
|
| 186 |
+
best = min(best, loss_acc)
|
| 187 |
+
|
| 188 |
+
# Snapshot for the in-page playground.
|
| 189 |
+
if (step % live_every == 0 or step == total - 1) and \
|
| 190 |
+
(time.time() - last_live > LIVE_MIN_SECONDS or step == total - 1):
|
| 191 |
+
try:
|
| 192 |
+
_write_live_checkpoint(model, job_id, step, tokenizer_dir,
|
| 193 |
+
first=not live_started)
|
| 194 |
+
live_started = True
|
| 195 |
+
last_live = time.time()
|
| 196 |
+
except Exception as exc:
|
| 197 |
+
log(f"live checkpoint skipped: {exc!r}")
|
| 198 |
+
|
| 199 |
+
if step % every == 0 or step == total - 1:
|
| 200 |
+
now = time.time()
|
| 201 |
+
rate = tok_win / max(now - t_win, 1e-6)
|
| 202 |
+
t_win, tok_win = now, 0
|
| 203 |
+
eta = (target_tokens - tokens_seen) / max(rate, 1.0)
|
| 204 |
+
db.add_metric(job_id, step, tokens_seen, loss_acc, lr)
|
| 205 |
+
db.update_job(
|
| 206 |
+
job_id, step=step, tokens_seen=tokens_seen, loss=loss_acc,
|
| 207 |
+
best_loss=best, lr=lr, tok_per_s=rate, eta_s=eta,
|
| 208 |
+
)
|
| 209 |
+
if step % (every * 4) == 0 or step == total - 1:
|
| 210 |
+
log(f"step {step:>6,}/{total:,} · loss {loss_acc:.4f} · "
|
| 211 |
+
f"ppl {math.exp(min(loss_acc, 20)):.1f} · lr {lr:.2e} · "
|
| 212 |
+
f"{rate / 1e3:.0f}k tok/s · ETA {eta / 60:.1f} min")
|
| 213 |
+
|
| 214 |
+
# ---- save ----
|
| 215 |
+
log("saving checkpoint...")
|
| 216 |
+
model.config.use_cache = True
|
| 217 |
+
model = model.float().cpu()
|
| 218 |
+
model.save_pretrained(out_dir, safe_serialization=True)
|
| 219 |
+
for fn in ("tokenizer.json", "tokenizer_config.json", "special_tokens_map.json"):
|
| 220 |
+
src = os.path.join(tokenizer_dir, fn)
|
| 221 |
+
if os.path.exists(src):
|
| 222 |
+
shutil.copy(src, os.path.join(out_dir, fn))
|
| 223 |
+
with open(os.path.join(out_dir, "generation_config.json"), "w") as f:
|
| 224 |
+
json.dump({"bos_token_id": 0, "eos_token_id": 0, "pad_token_id": 1,
|
| 225 |
+
"_from_model_config": True}, f, indent=2)
|
| 226 |
+
with open(os.path.join(out_dir, "training_run.json"), "w") as f:
|
| 227 |
+
json.dump({
|
| 228 |
+
"job_id": job_id, "tier": job["tier"], "label": tier.label,
|
| 229 |
+
"n_params": n_params, "target_tokens": target_tokens,
|
| 230 |
+
"tokens_seen": tokens_seen, "steps": total, "seq_len": seq_len,
|
| 231 |
+
"final_loss": loss_acc, "best_loss": best,
|
| 232 |
+
"peak_lr": tier.lr, "batch_tokens": tier.batch_tokens,
|
| 233 |
+
"optimizer": "AdamW(0.9, 0.95) wd=0.1 clip=1.0",
|
| 234 |
+
"schedule": "warmup 2% + cosine to 10%",
|
| 235 |
+
"dataset": "HuggingFaceFW/fineweb-edu (sample-10BT)",
|
| 236 |
+
"architecture": "LlamaForCausalLM (SiLU, RMSNorm, RoPE, GQA, tied embeddings)",
|
| 237 |
+
"wall_time_s": time.time() - t_start,
|
| 238 |
+
"trained_by": job["username"],
|
| 239 |
+
}, f, indent=2)
|
| 240 |
+
|
| 241 |
+
db.update_job(job_id, status="done", finished_at=time.time(),
|
| 242 |
+
tokens_seen=tokens_seen, loss=loss_acc, best_loss=best,
|
| 243 |
+
step=total, eta_s=0)
|
| 244 |
+
|
| 245 |
+
# Mirror the finished run to the bucket so it survives the next rebuild.
|
| 246 |
+
try:
|
| 247 |
+
payload = db.export_run(job_id)
|
| 248 |
+
if payload:
|
| 249 |
+
store.save_run(job_id, out_dir, payload, log)
|
| 250 |
+
except Exception as exc:
|
| 251 |
+
log(f"could not archive the run to durable storage: {exc!r}")
|
| 252 |
+
log(f"training finished in {(time.time() - t_start) / 60:.1f} min · "
|
| 253 |
+
f"final loss {loss_acc:.4f}")
|
| 254 |
+
return out_dir
|
nanodex/worker.py
ADDED
|
@@ -0,0 +1,137 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""One worker thread per GPU, pulling jobs off the shared queue."""
|
| 2 |
+
|
| 3 |
+
import threading
|
| 4 |
+
import time
|
| 5 |
+
import traceback
|
| 6 |
+
|
| 7 |
+
from . import db
|
| 8 |
+
from .trainer import run_training
|
| 9 |
+
|
| 10 |
+
_cancel = set()
|
| 11 |
+
_cancel_lock = threading.Lock()
|
| 12 |
+
_workers = []
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
def request_cancel(job_id):
|
| 16 |
+
with _cancel_lock:
|
| 17 |
+
_cancel.add(job_id)
|
| 18 |
+
j = db.get_job(job_id)
|
| 19 |
+
if j and j["status"] == "queued":
|
| 20 |
+
db.update_job(job_id, status="cancelled", finished_at=time.time())
|
| 21 |
+
return True
|
| 22 |
+
return bool(j)
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
def _is_cancelled(job_id):
|
| 26 |
+
with _cancel_lock:
|
| 27 |
+
return job_id in _cancel
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
_devices_cache = None
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
def devices():
|
| 34 |
+
"""Never raises and never blocks twice — CUDA probing is done once."""
|
| 35 |
+
global _devices_cache
|
| 36 |
+
if _devices_cache is not None:
|
| 37 |
+
return _devices_cache
|
| 38 |
+
found = ["cpu"]
|
| 39 |
+
try:
|
| 40 |
+
import torch
|
| 41 |
+
if torch.cuda.is_available() and torch.cuda.device_count() > 0:
|
| 42 |
+
found = [f"cuda:{i}" for i in range(torch.cuda.device_count())]
|
| 43 |
+
except Exception as exc:
|
| 44 |
+
print(f"[worker] no CUDA ({exc!r}) — falling back to CPU", flush=True)
|
| 45 |
+
_devices_cache = found
|
| 46 |
+
return found
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
def _loop(gpu_index, device, tokenizer_dir):
|
| 50 |
+
while True:
|
| 51 |
+
try:
|
| 52 |
+
job = db.claim_next_queued(gpu_index)
|
| 53 |
+
except Exception:
|
| 54 |
+
job = None
|
| 55 |
+
if not job:
|
| 56 |
+
time.sleep(2)
|
| 57 |
+
continue
|
| 58 |
+
|
| 59 |
+
jid = job["id"]
|
| 60 |
+
|
| 61 |
+
def log(msg, _jid=jid, _dev=device):
|
| 62 |
+
print(f"[{_dev}][{_jid}] {msg}", flush=True)
|
| 63 |
+
try:
|
| 64 |
+
db.add_log(_jid, msg)
|
| 65 |
+
except Exception:
|
| 66 |
+
pass
|
| 67 |
+
|
| 68 |
+
# Don't start on a barely-filled cache, or a 300M-token run would
|
| 69 |
+
# loop over the same few million tokens hundreds of times.
|
| 70 |
+
try:
|
| 71 |
+
from . import data as ndata
|
| 72 |
+
need = min(int(job["target_tokens"]), ndata.TARGET_TOKENS)
|
| 73 |
+
last_note = 0.0
|
| 74 |
+
while ndata.available_tokens() < need and ndata.state()["status"] == "building":
|
| 75 |
+
if time.time() - last_note > 60:
|
| 76 |
+
log(f"waiting for the fineweb-edu cache to reach "
|
| 77 |
+
f"{need / 1e6:.0f}M tokens "
|
| 78 |
+
f"({ndata.available_tokens() / 1e6:.0f}M ready so far — "
|
| 79 |
+
f"this is a one-time cost per container)")
|
| 80 |
+
last_note = time.time()
|
| 81 |
+
if _is_cancelled(jid):
|
| 82 |
+
break
|
| 83 |
+
time.sleep(5)
|
| 84 |
+
except Exception:
|
| 85 |
+
pass
|
| 86 |
+
|
| 87 |
+
if _is_cancelled(jid):
|
| 88 |
+
db.update_job(jid, status="cancelled", finished_at=time.time())
|
| 89 |
+
with _cancel_lock:
|
| 90 |
+
_cancel.discard(jid)
|
| 91 |
+
continue
|
| 92 |
+
|
| 93 |
+
log(f"starting on {device}")
|
| 94 |
+
try:
|
| 95 |
+
run_training(job, device, tokenizer_dir, lambda: _is_cancelled(jid), log)
|
| 96 |
+
except Exception as exc:
|
| 97 |
+
tb = traceback.format_exc(limit=6)
|
| 98 |
+
print(tb, flush=True)
|
| 99 |
+
log(f"FAILED: {exc!r}")
|
| 100 |
+
db.update_job(jid, status="failed", error=str(exc)[:500],
|
| 101 |
+
finished_at=time.time())
|
| 102 |
+
finally:
|
| 103 |
+
with _cancel_lock:
|
| 104 |
+
_cancel.discard(jid)
|
| 105 |
+
try:
|
| 106 |
+
import torch
|
| 107 |
+
if device.startswith("cuda"):
|
| 108 |
+
torch.cuda.empty_cache()
|
| 109 |
+
except Exception:
|
| 110 |
+
pass
|
| 111 |
+
|
| 112 |
+
|
| 113 |
+
def start_workers(tokenizer_dir):
|
| 114 |
+
global _workers
|
| 115 |
+
if _workers:
|
| 116 |
+
return _workers
|
| 117 |
+
for i, dev in enumerate(devices()):
|
| 118 |
+
if dev.startswith("cuda"):
|
| 119 |
+
try:
|
| 120 |
+
import torch
|
| 121 |
+
name = torch.cuda.get_device_name(i)
|
| 122 |
+
gb = torch.cuda.get_device_properties(i).total_memory / 1024**3
|
| 123 |
+
print(f"[worker] {dev}: {name} ({gb:.0f} GB)", flush=True)
|
| 124 |
+
except Exception:
|
| 125 |
+
pass
|
| 126 |
+
t = threading.Thread(target=_loop, args=(i, dev, tokenizer_dir),
|
| 127 |
+
name=f"worker-{dev}", daemon=True)
|
| 128 |
+
t.start()
|
| 129 |
+
_workers.append(t)
|
| 130 |
+
print(f"[worker] started on {dev}", flush=True)
|
| 131 |
+
return _workers
|
| 132 |
+
|
| 133 |
+
|
| 134 |
+
def worker_info():
|
| 135 |
+
devs = devices()
|
| 136 |
+
running = {j["gpu_index"]: j for j in db.list_jobs(status=["running"], limit=64)}
|
| 137 |
+
return [{"index": i, "device": d, "job": running.get(i)} for i, d in enumerate(devs)]
|
requirements.txt
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
fastapi>=0.115
|
| 2 |
+
uvicorn[standard]>=0.32
|
| 3 |
+
jinja2>=3.1
|
| 4 |
+
python-multipart>=0.0.12
|
| 5 |
+
itsdangerous>=2.2
|
| 6 |
+
authlib>=1.3
|
| 7 |
+
httpx>=0.27
|
| 8 |
+
|
| 9 |
+
torch>=2.6
|
| 10 |
+
transformers>=4.48
|
| 11 |
+
datasets>=3.2
|
| 12 |
+
tokenizers>=0.21
|
| 13 |
+
safetensors>=0.4
|
| 14 |
+
huggingface_hub>=0.27
|
| 15 |
+
numpy>=1.26
|
tokenizer/special_tokens_map.json
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"bos_token": "<|endoftext|>",
|
| 3 |
+
"eos_token": "<|endoftext|>",
|
| 4 |
+
"pad_token": "<|pad|>"
|
| 5 |
+
}
|
tokenizer/tokenizer.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
tokenizer/tokenizer_config.json
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"added_tokens_decoder": {
|
| 3 |
+
"0": {
|
| 4 |
+
"content": "<|endoftext|>",
|
| 5 |
+
"lstrip": false,
|
| 6 |
+
"normalized": false,
|
| 7 |
+
"rstrip": false,
|
| 8 |
+
"single_word": false,
|
| 9 |
+
"special": true
|
| 10 |
+
},
|
| 11 |
+
"1": {
|
| 12 |
+
"content": "<|pad|>",
|
| 13 |
+
"lstrip": false,
|
| 14 |
+
"normalized": false,
|
| 15 |
+
"rstrip": false,
|
| 16 |
+
"single_word": false,
|
| 17 |
+
"special": true
|
| 18 |
+
}
|
| 19 |
+
},
|
| 20 |
+
"bos_token": "<|endoftext|>",
|
| 21 |
+
"eos_token": "<|endoftext|>",
|
| 22 |
+
"pad_token": "<|pad|>",
|
| 23 |
+
"unk_token": null,
|
| 24 |
+
"clean_up_tokenization_spaces": false,
|
| 25 |
+
"model_max_length": 512,
|
| 26 |
+
"tokenizer_class": "PreTrainedTokenizerFast"
|
| 27 |
+
}
|
web/static/app.css
ADDED
|
@@ -0,0 +1,351 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
:root{
|
| 2 |
+
--bg:#080b0f; --bg-2:#0d1117; --panel:#11161d; --panel-2:#161d26;
|
| 3 |
+
--line:#1e2733; --line-2:#2a3441;
|
| 4 |
+
--fg:#e7edf5; --fg-2:#9aa8ba; --fg-3:#64748b;
|
| 5 |
+
--accent:#2ee6a8; --accent-2:#19b884; --accent-dim:#2ee6a81a;
|
| 6 |
+
--blue:#5aa9ff; --violet:#a78bfa; --amber:#fbbf24; --red:#fb7185;
|
| 7 |
+
--r:14px; --r-sm:9px;
|
| 8 |
+
--mono:ui-monospace,"SF Mono",Menlo,Consolas,monospace;
|
| 9 |
+
--sans:-apple-system,BlinkMacSystemFont,"Inter","Segoe UI",Roboto,sans-serif;
|
| 10 |
+
}
|
| 11 |
+
*{box-sizing:border-box}
|
| 12 |
+
html,body{margin:0;padding:0}
|
| 13 |
+
body{background:var(--bg);color:var(--fg);font-family:var(--sans);
|
| 14 |
+
font-size:15px;line-height:1.6;-webkit-font-smoothing:antialiased;
|
| 15 |
+
min-height:100vh;display:flex;flex-direction:column}
|
| 16 |
+
a{color:inherit;text-decoration:none}
|
| 17 |
+
h1,h2,h3,h4{margin:0;line-height:1.2;letter-spacing:-.022em;font-weight:650}
|
| 18 |
+
::selection{background:var(--accent);color:#04140e}
|
| 19 |
+
|
| 20 |
+
/* ---------- layout ---------- */
|
| 21 |
+
.wrap{width:100%;max-width:1140px;margin:0 auto;padding:0 24px}
|
| 22 |
+
main{flex:1;padding:36px 0 72px}
|
| 23 |
+
.grid{display:grid;gap:16px}
|
| 24 |
+
.g2{grid-template-columns:repeat(2,1fr)}
|
| 25 |
+
.g3{grid-template-columns:repeat(3,1fr)}
|
| 26 |
+
.g4{grid-template-columns:repeat(4,1fr)}
|
| 27 |
+
@media(max-width:880px){.g2,.g3,.g4{grid-template-columns:1fr}}
|
| 28 |
+
|
| 29 |
+
/* ---------- nav ---------- */
|
| 30 |
+
nav{position:sticky;top:0;z-index:50;background:rgba(8,11,15,.82);
|
| 31 |
+
backdrop-filter:blur(14px);border-bottom:1px solid var(--line)}
|
| 32 |
+
nav .wrap{display:flex;align-items:center;gap:8px;height:60px}
|
| 33 |
+
.brand{display:flex;align-items:center;gap:9px;font-weight:700;font-size:16.5px;
|
| 34 |
+
letter-spacing:-.03em;margin-right:14px}
|
| 35 |
+
.brand .dot{width:9px;height:9px;border-radius:50%;background:var(--accent);
|
| 36 |
+
box-shadow:0 0 12px var(--accent)}
|
| 37 |
+
.navlinks{display:flex;gap:2px;flex:1}
|
| 38 |
+
.navlinks a{padding:7px 13px;border-radius:8px;color:var(--fg-2);font-size:14px;
|
| 39 |
+
font-weight:500;transition:.15s}
|
| 40 |
+
.navlinks a:hover{color:var(--fg);background:var(--panel)}
|
| 41 |
+
.navlinks a.on{color:var(--accent);background:var(--accent-dim)}
|
| 42 |
+
.avatar{width:30px;height:30px;border-radius:50%;border:1px solid var(--line-2);
|
| 43 |
+
object-fit:cover;background:var(--panel-2)}
|
| 44 |
+
.usermenu{display:flex;align-items:center;gap:9px;padding:4px 11px 4px 4px;
|
| 45 |
+
border:1px solid var(--line);border-radius:99px;font-size:13.5px;font-weight:500}
|
| 46 |
+
.usermenu:hover{border-color:var(--line-2);background:var(--panel)}
|
| 47 |
+
@media(max-width:760px){.navlinks a{padding:7px 9px;font-size:13px}
|
| 48 |
+
.usermenu span{display:none}}
|
| 49 |
+
|
| 50 |
+
/* ---------- buttons ---------- */
|
| 51 |
+
.btn{display:inline-flex;align-items:center;justify-content:center;gap:8px;
|
| 52 |
+
padding:10px 18px;border-radius:var(--r-sm);border:1px solid var(--line-2);
|
| 53 |
+
background:var(--panel);color:var(--fg);font:inherit;font-size:14px;
|
| 54 |
+
font-weight:560;cursor:pointer;transition:.15s;white-space:nowrap}
|
| 55 |
+
.btn:hover{background:var(--panel-2);border-color:#374151}
|
| 56 |
+
.btn:disabled{opacity:.4;cursor:not-allowed}
|
| 57 |
+
.btn.primary{background:var(--accent);border-color:var(--accent);color:#04140e}
|
| 58 |
+
.btn.primary:hover{background:#48f0b8;border-color:#48f0b8}
|
| 59 |
+
.btn.ghost{background:transparent;border-color:transparent;color:var(--fg-2)}
|
| 60 |
+
.btn.ghost:hover{color:var(--fg);background:var(--panel)}
|
| 61 |
+
.btn.danger{border-color:#4c1d24;color:var(--red);background:transparent}
|
| 62 |
+
.btn.danger:hover{background:#2a1015}
|
| 63 |
+
.btn.lg{padding:13px 26px;font-size:15px}
|
| 64 |
+
.btn.sm{padding:6px 12px;font-size:13px}
|
| 65 |
+
|
| 66 |
+
/* ---------- cards ---------- */
|
| 67 |
+
.card{background:var(--panel);border:1px solid var(--line);border-radius:var(--r);
|
| 68 |
+
padding:20px}
|
| 69 |
+
.card.pad-lg{padding:26px}
|
| 70 |
+
.card h3{font-size:16px;margin-bottom:3px}
|
| 71 |
+
.sub{color:var(--fg-2);font-size:13.5px}
|
| 72 |
+
.muted{color:var(--fg-3)}
|
| 73 |
+
.mono{font-family:var(--mono);font-variant-numeric:tabular-nums}
|
| 74 |
+
.eyebrow{font-size:11px;text-transform:uppercase;letter-spacing:.11em;
|
| 75 |
+
color:var(--fg-3);font-weight:650}
|
| 76 |
+
.sect-h{display:flex;align-items:baseline;justify-content:space-between;
|
| 77 |
+
gap:12px;margin:0 0 14px}
|
| 78 |
+
.sect-h h2{font-size:19px}
|
| 79 |
+
|
| 80 |
+
/* ---------- chips ---------- */
|
| 81 |
+
.chip{display:inline-flex;align-items:center;gap:6px;font-size:11.5px;
|
| 82 |
+
font-weight:650;padding:3px 10px;border-radius:99px;letter-spacing:.02em;
|
| 83 |
+
color:var(--c);background:color-mix(in srgb,var(--c) 13%,transparent);
|
| 84 |
+
border:1px solid color-mix(in srgb,var(--c) 30%,transparent);white-space:nowrap}
|
| 85 |
+
.chip i{width:5px;height:5px;border-radius:50%;background:var(--c);flex:none}
|
| 86 |
+
.chip.queued{--c:var(--violet)} .chip.running{--c:var(--accent)}
|
| 87 |
+
.chip.done{--c:var(--blue)} .chip.failed{--c:var(--red)}
|
| 88 |
+
.chip.cancelled{--c:#64748b}
|
| 89 |
+
.chip.running i{animation:pulse 1.4s ease-in-out infinite}
|
| 90 |
+
@keyframes pulse{0%,100%{opacity:1}50%{opacity:.25}}
|
| 91 |
+
|
| 92 |
+
/* ---------- progress ---------- */
|
| 93 |
+
.bar{height:6px;border-radius:99px;background:var(--line);overflow:hidden}
|
| 94 |
+
.bar>i{display:block;height:100%;border-radius:99px;
|
| 95 |
+
background:linear-gradient(90deg,var(--accent-2),var(--accent));
|
| 96 |
+
transition:width .5s ease}
|
| 97 |
+
.bar.thin{height:4px}
|
| 98 |
+
|
| 99 |
+
/* ---------- stat tiles ---------- */
|
| 100 |
+
.stat{background:var(--panel);border:1px solid var(--line);border-radius:var(--r);
|
| 101 |
+
padding:16px 18px}
|
| 102 |
+
.stat b{display:block;font-size:25px;font-weight:680;letter-spacing:-.03em;
|
| 103 |
+
font-variant-numeric:tabular-nums;line-height:1.15}
|
| 104 |
+
.stat span{font-size:11px;text-transform:uppercase;letter-spacing:.1em;
|
| 105 |
+
color:var(--fg-3);font-weight:650}
|
| 106 |
+
.stat small{display:block;color:var(--fg-3);font-size:12px;margin-top:2px}
|
| 107 |
+
|
| 108 |
+
/* ---------- run rows ---------- */
|
| 109 |
+
.runs{display:flex;flex-direction:column;gap:9px}
|
| 110 |
+
.run{display:block;background:var(--panel);border:1px solid var(--line);
|
| 111 |
+
border-radius:var(--r);padding:15px 18px;transition:.15s}
|
| 112 |
+
.run:hover{border-color:var(--line-2);background:var(--panel-2)}
|
| 113 |
+
.run-top{display:flex;align-items:center;gap:11px;margin-bottom:4px}
|
| 114 |
+
.run-top h4{font-size:15px;font-weight:620;flex:1;min-width:0;
|
| 115 |
+
overflow:hidden;text-overflow:ellipsis;white-space:nowrap}
|
| 116 |
+
.run-meta{display:flex;gap:14px;flex-wrap:wrap;font-size:12.5px;color:var(--fg-3);
|
| 117 |
+
font-variant-numeric:tabular-nums}
|
| 118 |
+
.run-meta b{color:var(--fg-2);font-weight:600}
|
| 119 |
+
.run .bar{margin-top:11px}
|
| 120 |
+
|
| 121 |
+
/* ---------- forms ---------- */
|
| 122 |
+
label.fl{display:block;font-size:12.5px;font-weight:620;color:var(--fg-2);
|
| 123 |
+
margin-bottom:7px}
|
| 124 |
+
input[type=text],input[type=number],textarea,select{width:100%;padding:11px 14px;
|
| 125 |
+
background:var(--bg-2);border:1px solid var(--line-2);border-radius:var(--r-sm);
|
| 126 |
+
color:var(--fg);font:inherit;font-size:14.5px;transition:.15s}
|
| 127 |
+
input:focus,textarea:focus,select:focus{outline:none;border-color:var(--accent);
|
| 128 |
+
box-shadow:0 0 0 3px var(--accent-dim)}
|
| 129 |
+
input::placeholder,textarea::placeholder{color:#4b5563}
|
| 130 |
+
textarea{resize:vertical;font-family:var(--mono);font-size:13.5px;line-height:1.65}
|
| 131 |
+
.hint{font-size:12.5px;color:var(--fg-3);margin-top:7px}
|
| 132 |
+
|
| 133 |
+
input[type=range]{-webkit-appearance:none;appearance:none;width:100%;height:6px;
|
| 134 |
+
background:var(--line);border-radius:99px;padding:0;border:none;outline:none}
|
| 135 |
+
input[type=range]::-webkit-slider-thumb{-webkit-appearance:none;width:20px;height:20px;
|
| 136 |
+
border-radius:50%;background:var(--accent);cursor:grab;border:3px solid var(--bg);
|
| 137 |
+
box-shadow:0 0 0 1px var(--accent)}
|
| 138 |
+
input[type=range]::-moz-range-thumb{width:20px;height:20px;border-radius:50%;
|
| 139 |
+
background:var(--accent);cursor:grab;border:3px solid var(--bg);box-shadow:0 0 0 1px var(--accent)}
|
| 140 |
+
|
| 141 |
+
/* ---------- pick cards ---------- */
|
| 142 |
+
.picks{display:grid;grid-template-columns:repeat(4,1fr);gap:11px}
|
| 143 |
+
@media(max-width:820px){.picks{grid-template-columns:repeat(2,1fr)}}
|
| 144 |
+
.pick{background:var(--bg-2);border:1.5px solid var(--line-2);border-radius:var(--r);
|
| 145 |
+
padding:16px;cursor:pointer;transition:.15s;text-align:left;color:inherit;
|
| 146 |
+
font:inherit;display:block}
|
| 147 |
+
.pick:hover{border-color:#3b4b5e;background:var(--panel-2)}
|
| 148 |
+
.pick.on{border-color:var(--accent);background:var(--accent-dim);
|
| 149 |
+
box-shadow:0 0 0 3px var(--accent-dim)}
|
| 150 |
+
.pick b{display:block;font-size:15.5px;font-weight:660;letter-spacing:-.02em}
|
| 151 |
+
.pick .p-n{font-family:var(--mono);font-size:12.5px;color:var(--accent);
|
| 152 |
+
margin:4px 0 8px;font-variant-numeric:tabular-nums}
|
| 153 |
+
.pick .p-d{font-size:12px;color:var(--fg-3);line-height:1.55}
|
| 154 |
+
|
| 155 |
+
/* ---------- steps ---------- */
|
| 156 |
+
.steps{display:flex;gap:0;margin-bottom:28px;border-bottom:1px solid var(--line)}
|
| 157 |
+
.stp{flex:1;padding:0 0 13px;border-bottom:2px solid transparent;
|
| 158 |
+
margin-bottom:-1px;color:var(--fg-3);transition:.2s}
|
| 159 |
+
.stp.on{color:var(--accent);border-bottom-color:var(--accent)}
|
| 160 |
+
.stp.ok{color:var(--fg-2);border-bottom-color:var(--line-2)}
|
| 161 |
+
.stp .n{font-family:var(--mono);font-size:11px;letter-spacing:.1em;
|
| 162 |
+
text-transform:uppercase;font-weight:700}
|
| 163 |
+
.stp .t{font-size:14px;font-weight:600;margin-top:2px}
|
| 164 |
+
@media(max-width:700px){.stp .t{font-size:12px}}
|
| 165 |
+
.panelstep{display:none} .panelstep.on{display:block;animation:fade .25s ease}
|
| 166 |
+
@keyframes fade{from{opacity:0;transform:translateY(6px)}to{opacity:1;transform:none}}
|
| 167 |
+
|
| 168 |
+
/* ---------- chart ---------- */
|
| 169 |
+
.chart{width:100%;height:280px;display:block}
|
| 170 |
+
.chart .gl{stroke:var(--line);stroke-width:1}
|
| 171 |
+
.chart .ax{fill:var(--fg-3);font-size:10.5px;font-family:var(--mono)}
|
| 172 |
+
.chart .ln{fill:none;stroke:var(--accent);stroke-width:2;
|
| 173 |
+
stroke-linejoin:round;stroke-linecap:round}
|
| 174 |
+
.chart .ar{fill:url(#lg)}
|
| 175 |
+
|
| 176 |
+
/* ---------- logs ---------- */
|
| 177 |
+
.logs{background:#05080b;border:1px solid var(--line);border-radius:var(--r-sm);
|
| 178 |
+
padding:14px 16px;font-family:var(--mono);font-size:12.2px;line-height:1.75;
|
| 179 |
+
max-height:340px;overflow:auto;color:#9fb3c8;white-space:pre-wrap;word-break:break-word}
|
| 180 |
+
.logs .t{color:#3f5063;margin-right:9px}
|
| 181 |
+
|
| 182 |
+
/* ---------- table ---------- */
|
| 183 |
+
table{width:100%;border-collapse:collapse;font-size:13.5px}
|
| 184 |
+
th{text-align:left;font-size:11px;text-transform:uppercase;letter-spacing:.09em;
|
| 185 |
+
color:var(--fg-3);font-weight:650;padding:0 14px 10px;border-bottom:1px solid var(--line)}
|
| 186 |
+
td{padding:13px 14px;border-bottom:1px solid var(--line);
|
| 187 |
+
font-variant-numeric:tabular-nums}
|
| 188 |
+
tbody tr:hover{background:var(--panel)}
|
| 189 |
+
tbody tr:last-child td{border-bottom:none}
|
| 190 |
+
|
| 191 |
+
/* ---------- landing ---------- */
|
| 192 |
+
.hero{padding:76px 0 56px;text-align:center;position:relative;overflow:hidden}
|
| 193 |
+
.hero::before{content:"";position:absolute;inset:-45% 15% auto;height:420px;
|
| 194 |
+
background:radial-gradient(ellipse at center,rgba(46,230,168,.13),transparent 65%);
|
| 195 |
+
pointer-events:none}
|
| 196 |
+
.hero h1{font-size:clamp(35px,6.4vw,60px);font-weight:720;letter-spacing:-.045em;
|
| 197 |
+
position:relative;line-height:1.05}
|
| 198 |
+
.hero h1 em{font-style:normal;
|
| 199 |
+
background:linear-gradient(100deg,var(--accent),var(--blue));
|
| 200 |
+
-webkit-background-clip:text;background-clip:text;-webkit-text-fill-color:transparent}
|
| 201 |
+
.hero p{font-size:clamp(15.5px,2.2vw,19px);color:var(--fg-2);max-width:640px;
|
| 202 |
+
margin:20px auto 0;position:relative}
|
| 203 |
+
.hero .cta{display:flex;gap:12px;justify-content:center;margin-top:34px;
|
| 204 |
+
position:relative;flex-wrap:wrap}
|
| 205 |
+
.tag{display:inline-flex;align-items:center;gap:8px;padding:5px 14px;
|
| 206 |
+
border:1px solid var(--line-2);border-radius:99px;font-size:12.5px;
|
| 207 |
+
color:var(--fg-2);margin-bottom:24px;position:relative;background:var(--panel)}
|
| 208 |
+
.tag b{color:var(--accent);font-weight:640}
|
| 209 |
+
|
| 210 |
+
.step-n{width:26px;height:26px;border-radius:8px;background:var(--accent-dim);
|
| 211 |
+
color:var(--accent);display:grid;place-items:center;font-family:var(--mono);
|
| 212 |
+
font-size:12px;font-weight:700;margin-bottom:11px}
|
| 213 |
+
|
| 214 |
+
/* ---------- misc ---------- */
|
| 215 |
+
.empty{padding:52px 24px;text-align:center;color:var(--fg-3);
|
| 216 |
+
border:1px dashed var(--line-2);border-radius:var(--r);font-size:14px}
|
| 217 |
+
.toast{position:fixed;bottom:22px;left:50%;transform:translateX(-50%) translateY(80px);
|
| 218 |
+
background:var(--panel-2);border:1px solid var(--line-2);border-radius:var(--r-sm);
|
| 219 |
+
padding:13px 20px;font-size:14px;z-index:200;opacity:0;transition:.28s;
|
| 220 |
+
box-shadow:0 12px 40px rgba(0,0,0,.55);max-width:min(560px,90vw)}
|
| 221 |
+
.toast.on{opacity:1;transform:translateX(-50%)}
|
| 222 |
+
.toast.err{border-color:#4c1d24;color:#ffc4cc}
|
| 223 |
+
.toast.ok{border-color:var(--accent-2)}
|
| 224 |
+
.gpu{background:var(--panel);border:1px solid var(--line);border-radius:var(--r);
|
| 225 |
+
padding:15px 17px}
|
| 226 |
+
.gpu.busy{border-color:color-mix(in srgb,var(--accent) 34%,var(--line))}
|
| 227 |
+
.gpu-h{display:flex;justify-content:space-between;align-items:baseline;
|
| 228 |
+
font-size:13.5px;font-weight:640;margin-bottom:9px}
|
| 229 |
+
.gpu-h span{font-family:var(--mono);font-size:11px;color:var(--fg-3)}
|
| 230 |
+
footer{border-top:1px solid var(--line);padding:26px 0;color:var(--fg-3);
|
| 231 |
+
font-size:13px}
|
| 232 |
+
footer .wrap{display:flex;justify-content:space-between;gap:16px;flex-wrap:wrap}
|
| 233 |
+
footer a{color:var(--fg-2)} footer a:hover{color:var(--accent)}
|
| 234 |
+
.kv{display:flex;justify-content:space-between;gap:14px;padding:9px 0;
|
| 235 |
+
border-bottom:1px solid var(--line);font-size:13.5px}
|
| 236 |
+
.kv:last-child{border-bottom:none}
|
| 237 |
+
.kv span{color:var(--fg-3)} .kv b{font-weight:600;font-variant-numeric:tabular-nums}
|
| 238 |
+
|
| 239 |
+
/* =========================================================================
|
| 240 |
+
Mobile / touch
|
| 241 |
+
========================================================================= */
|
| 242 |
+
html{-webkit-text-size-adjust:100%}
|
| 243 |
+
body{overflow-x:hidden}
|
| 244 |
+
.tscroll{overflow-x:auto;-webkit-overflow-scrolling:touch}
|
| 245 |
+
.logs{-webkit-overflow-scrolling:touch}
|
| 246 |
+
img,svg{max-width:100%}
|
| 247 |
+
|
| 248 |
+
/* Stat tiles stay two-up rather than becoming a tall stack. */
|
| 249 |
+
@media(max-width:880px){
|
| 250 |
+
.g4{grid-template-columns:repeat(2,1fr)}
|
| 251 |
+
.g3{grid-template-columns:repeat(2,1fr)}
|
| 252 |
+
.stats{grid-template-columns:repeat(2,1fr)}
|
| 253 |
+
}
|
| 254 |
+
|
| 255 |
+
@media(max-width:760px){
|
| 256 |
+
.wrap{padding:0 16px}
|
| 257 |
+
main{padding:26px 0 56px}
|
| 258 |
+
|
| 259 |
+
/* Two-row nav: brand + account on top, scrollable links underneath. */
|
| 260 |
+
nav .wrap{height:auto;flex-wrap:wrap;padding-top:10px;padding-bottom:8px;
|
| 261 |
+
row-gap:6px}
|
| 262 |
+
.brand{margin-right:auto;font-size:16px}
|
| 263 |
+
.navlinks{order:3;flex-basis:100%;overflow-x:auto;
|
| 264 |
+
-webkit-overflow-scrolling:touch;scrollbar-width:none;gap:3px;
|
| 265 |
+
padding-bottom:2px}
|
| 266 |
+
.navlinks::-webkit-scrollbar{display:none}
|
| 267 |
+
.navlinks a{padding:7px 12px;font-size:13.5px;flex:none;white-space:nowrap}
|
| 268 |
+
/* "Sign in with Hugging Face" is too wide for a phone header. */
|
| 269 |
+
.signin-long{display:none}
|
| 270 |
+
nav .btn.primary{padding:9px 14px;min-height:36px}
|
| 271 |
+
|
| 272 |
+
.hero{padding:44px 0 38px}
|
| 273 |
+
.hero .cta{flex-direction:column;align-items:stretch}
|
| 274 |
+
.hero .cta .btn{width:100%}
|
| 275 |
+
|
| 276 |
+
.btn{padding:11px 17px;min-height:42px}
|
| 277 |
+
.btn.sm{padding:8px 13px;min-height:36px}
|
| 278 |
+
.btn.lg{padding:14px 22px;min-height:48px}
|
| 279 |
+
|
| 280 |
+
.card{padding:17px} .card.pad-lg{padding:19px}
|
| 281 |
+
.stat{padding:14px 15px}
|
| 282 |
+
.stat b{font-size:21px}
|
| 283 |
+
|
| 284 |
+
.sect-h{flex-direction:column;align-items:flex-start;gap:8px}
|
| 285 |
+
.sect-h h1,.sect-h h2{font-size:18px}
|
| 286 |
+
|
| 287 |
+
/* Wizard: four steps still fit by shrinking the labels. */
|
| 288 |
+
.steps{margin-bottom:22px}
|
| 289 |
+
.stp{padding-bottom:11px}
|
| 290 |
+
.stp .n{font-size:9.5px;letter-spacing:.06em}
|
| 291 |
+
.stp .t{font-size:12px}
|
| 292 |
+
|
| 293 |
+
.chart{height:210px}
|
| 294 |
+
.run{padding:14px 15px}
|
| 295 |
+
.run-meta{gap:10px;font-size:12px}
|
| 296 |
+
.kv{font-size:13px;gap:10px}
|
| 297 |
+
input[type=text],input[type=number],textarea,select{font-size:16px} /* no iOS zoom */
|
| 298 |
+
input[type=range]{height:8px}
|
| 299 |
+
input[type=range]::-webkit-slider-thumb{width:24px;height:24px}
|
| 300 |
+
input[type=range]::-moz-range-thumb{width:24px;height:24px}
|
| 301 |
+
.logs{font-size:11.6px;max-height:260px}
|
| 302 |
+
th,td{padding-left:10px;padding-right:10px}
|
| 303 |
+
.toast{left:16px;right:16px;transform:translateY(80px);max-width:none}
|
| 304 |
+
.toast.on{transform:none}
|
| 305 |
+
}
|
| 306 |
+
|
| 307 |
+
@media(max-width:520px){
|
| 308 |
+
.picks{grid-template-columns:1fr}
|
| 309 |
+
.g3{grid-template-columns:1fr}
|
| 310 |
+
.pick{display:flex;align-items:baseline;gap:10px;flex-wrap:wrap;padding:14px}
|
| 311 |
+
.pick b{flex:none}
|
| 312 |
+
.pick .p-n{margin:0}
|
| 313 |
+
.pick .p-d{flex-basis:100%;margin-top:4px}
|
| 314 |
+
.hero h1{letter-spacing:-.035em}
|
| 315 |
+
footer .wrap{flex-direction:column;gap:8px;font-size:12.5px}
|
| 316 |
+
}
|
| 317 |
+
|
| 318 |
+
@media(max-width:359px){
|
| 319 |
+
.g4,.stats{grid-template-columns:1fr}
|
| 320 |
+
.stat b{font-size:24px}
|
| 321 |
+
}
|
| 322 |
+
|
| 323 |
+
@media(hover:none){
|
| 324 |
+
.run:active{background:var(--panel-2)}
|
| 325 |
+
.pick:active{border-color:var(--accent)}
|
| 326 |
+
}
|
| 327 |
+
|
| 328 |
+
/* Long labels that collapse on narrow screens */
|
| 329 |
+
.signin-short{display:none}
|
| 330 |
+
@media(max-width:760px){ .signin-short{display:inline} }
|
| 331 |
+
|
| 332 |
+
/* Empty-state text inside the loss chart needs to survive the viewBox scale */
|
| 333 |
+
.chart .ph{fill:var(--fg-3);font-size:15px;font-family:var(--sans)}
|
| 334 |
+
|
| 335 |
+
/* Playground output — prompt dimmed, continuation bright */
|
| 336 |
+
.pgout{background:#05080b;border:1px solid var(--line);border-radius:var(--r-sm);
|
| 337 |
+
padding:14px 16px;font-family:var(--mono);font-size:13px;line-height:1.8;
|
| 338 |
+
white-space:pre-wrap;word-break:break-word;max-height:340px;overflow:auto;
|
| 339 |
+
-webkit-overflow-scrolling:touch}
|
| 340 |
+
.pgout .pgp{color:var(--fg-3)}
|
| 341 |
+
.pgout .pgc{color:#bfe9d8}
|
| 342 |
+
@media(max-width:760px){ .pgout{font-size:12.4px;padding:12px 13px} }
|
| 343 |
+
|
| 344 |
+
/* Shown only when the app is running inside the Space iframe */
|
| 345 |
+
.frame-note{display:none}
|
| 346 |
+
.framed .frame-note{display:block;position:relative;margin:16px auto 0;
|
| 347 |
+
max-width:460px;font-size:12.5px;color:var(--fg-3);text-align:center}
|
| 348 |
+
|
| 349 |
+
/* Token-budget shortcuts under the slider */
|
| 350 |
+
.quickpick{display:flex;gap:7px;margin-top:14px;flex-wrap:wrap}
|
| 351 |
+
.quickpick .btn{flex:1;min-width:64px}
|
web/static/app.js
ADDED
|
@@ -0,0 +1,164 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
/* NanoDex — shared frontend helpers */
|
| 2 |
+
|
| 3 |
+
const NX = {
|
| 4 |
+
fmtTokens(n){ n = Number(n||0);
|
| 5 |
+
if(n>=1e9) return (n/1e9).toFixed(2)+'B';
|
| 6 |
+
if(n>=1e6) return (n/1e6).toFixed(n>=1e7?0:1).replace(/\.0$/,'')+'M';
|
| 7 |
+
if(n>=1e3) return Math.round(n/1e3)+'k';
|
| 8 |
+
return String(n); },
|
| 9 |
+
fmtParams(n){ n = Number(n||0);
|
| 10 |
+
return n>=1e6 ? (n/1e6).toFixed(2)+'M' : Math.round(n/1e3)+'k'; },
|
| 11 |
+
fmtTime(s){ s = Number(s||0); if(!isFinite(s)||s<0) return '—';
|
| 12 |
+
if(s<90) return Math.round(s)+'s';
|
| 13 |
+
if(s<5400) return Math.round(s/60)+' min';
|
| 14 |
+
return (s/3600).toFixed(1)+' h'; },
|
| 15 |
+
ago(ts){ if(!ts) return '—';
|
| 16 |
+
const d = Date.now()/1000 - ts;
|
| 17 |
+
if(d<60) return 'just now';
|
| 18 |
+
if(d<3600) return Math.floor(d/60)+'m ago';
|
| 19 |
+
if(d<86400) return Math.floor(d/3600)+'h ago';
|
| 20 |
+
return Math.floor(d/86400)+'d ago'; },
|
| 21 |
+
esc(s){ const d=document.createElement('div'); d.textContent=s==null?'':String(s);
|
| 22 |
+
return d.innerHTML; },
|
| 23 |
+
|
| 24 |
+
chip(status, extra){
|
| 25 |
+
const label = {queued:'queued', running:'training', done:'completed',
|
| 26 |
+
failed:'failed', cancelled:'cancelled'}[status] || status;
|
| 27 |
+
return `<span class="chip ${status}"><i></i>${label}${extra?' · '+NX.esc(extra):''}</span>`;
|
| 28 |
+
},
|
| 29 |
+
|
| 30 |
+
async get(url){
|
| 31 |
+
const r = await fetch(url, {headers:{'Accept':'application/json'}});
|
| 32 |
+
if(!r.ok) throw new Error((await r.json().catch(()=>({}))).error || r.statusText);
|
| 33 |
+
return r.json();
|
| 34 |
+
},
|
| 35 |
+
async post(url, body){
|
| 36 |
+
const r = await fetch(url, {method:'POST',
|
| 37 |
+
headers:{'Content-Type':'application/json'}, body: JSON.stringify(body||{})});
|
| 38 |
+
const j = await r.json().catch(()=>({}));
|
| 39 |
+
if(!r.ok) throw new Error(j.error || r.statusText);
|
| 40 |
+
return j;
|
| 41 |
+
},
|
| 42 |
+
|
| 43 |
+
toast(msg, kind){
|
| 44 |
+
let t = document.querySelector('.toast');
|
| 45 |
+
if(!t){ t = document.createElement('div'); t.className='toast';
|
| 46 |
+
document.body.appendChild(t); }
|
| 47 |
+
t.className = 'toast ' + (kind||'');
|
| 48 |
+
t.innerHTML = msg;
|
| 49 |
+
requestAnimationFrame(()=> t.classList.add('on'));
|
| 50 |
+
clearTimeout(t._h);
|
| 51 |
+
t._h = setTimeout(()=> t.classList.remove('on'), 5200);
|
| 52 |
+
},
|
| 53 |
+
|
| 54 |
+
runRow(j){
|
| 55 |
+
const pct = (j.progress*100).toFixed(1);
|
| 56 |
+
const bits = [];
|
| 57 |
+
if(j.status==='running'){
|
| 58 |
+
bits.push(`<span><b>${pct}%</b> of ${NX.fmtTokens(j.target_tokens)}</span>`);
|
| 59 |
+
if(j.loss) bits.push(`<span>loss <b>${j.loss.toFixed(4)}</b></span>`);
|
| 60 |
+
bits.push(`<span><b>${Math.round(j.tok_per_s/1e3)}k</b> tok/s</span>`);
|
| 61 |
+
if(j.eta_s) bits.push(`<span><b>${NX.fmtTime(j.eta_s)}</b> left</span>`);
|
| 62 |
+
} else if(j.status==='queued'){
|
| 63 |
+
bits.push(`<span>position <b>#${j.queue_position||'?'}</b></span>`);
|
| 64 |
+
bits.push(`<span><b>${NX.fmtTokens(j.target_tokens)}</b> tokens queued</span>`);
|
| 65 |
+
} else if(j.status==='done'){
|
| 66 |
+
bits.push(`<span><b>${NX.fmtTokens(j.tokens_seen)}</b> tokens</span>`);
|
| 67 |
+
if(j.loss) bits.push(`<span>final loss <b>${j.loss.toFixed(4)}</b></span>`);
|
| 68 |
+
bits.push(`<span>${j.pushed_repo ? 'published' : 'not published'}</span>`);
|
| 69 |
+
} else {
|
| 70 |
+
bits.push(`<span>${NX.esc(j.error||'—')}</span>`);
|
| 71 |
+
}
|
| 72 |
+
const showBar = j.status==='running';
|
| 73 |
+
return `<a class="run" href="/models/${j.id}">
|
| 74 |
+
<div class="run-top">
|
| 75 |
+
<h4>${NX.esc(j.name)}</h4>
|
| 76 |
+
${NX.chip(j.status)}
|
| 77 |
+
</div>
|
| 78 |
+
<div class="run-meta">
|
| 79 |
+
<span class="mono">${NX.esc(j.tier_label)}</span>
|
| 80 |
+
<span>${Number(j.n_params).toLocaleString()} params</span>
|
| 81 |
+
${bits.join('')}
|
| 82 |
+
<span class="muted">${NX.ago(j.created_at)}</span>
|
| 83 |
+
</div>
|
| 84 |
+
${showBar ? `<div class="bar"><i style="width:${pct}%"></i></div>` : ''}
|
| 85 |
+
</a>`;
|
| 86 |
+
},
|
| 87 |
+
|
| 88 |
+
/* Minimal dependency-free loss chart.
|
| 89 |
+
The viewBox is set to the element's real pixel size so that 1 user unit ==
|
| 90 |
+
1 CSS pixel: a fixed 760-wide viewBox squeezed into a 280px phone would
|
| 91 |
+
render the axis labels at ~4px. */
|
| 92 |
+
lossChart(el, pts){
|
| 93 |
+
if(!el) return;
|
| 94 |
+
el._pts = pts;
|
| 95 |
+
const rect = el.getBoundingClientRect();
|
| 96 |
+
const W = Math.max(280, Math.round(rect.width) || 760);
|
| 97 |
+
const H = Math.max(170, Math.round(rect.height) || 280);
|
| 98 |
+
const narrow = W < 460;
|
| 99 |
+
const PL = narrow ? 40 : 54, PR = narrow ? 8 : 14;
|
| 100 |
+
const PT = 16, PB = narrow ? 28 : 34;
|
| 101 |
+
const TICKS = narrow ? 3 : 4;
|
| 102 |
+
el.setAttribute('viewBox', `0 0 ${W} ${H}`);
|
| 103 |
+
if(!pts || pts.length < 2){
|
| 104 |
+
el.innerHTML = `<text x="${W/2}" y="${H/2}" class="ph"
|
| 105 |
+
text-anchor="middle">waiting for the first metrics…</text>`;
|
| 106 |
+
return;
|
| 107 |
+
}
|
| 108 |
+
const xs = pts.map(p=>p.tokens), ys = pts.map(p=>p.loss);
|
| 109 |
+
const x0=Math.min(...xs), x1=Math.max(...xs);
|
| 110 |
+
let y0=Math.min(...ys), y1=Math.max(...ys);
|
| 111 |
+
const padY=(y1-y0)*0.12 || 0.4; y0-=padY; y1+=padY;
|
| 112 |
+
const sx = v => PL + (v-x0)/((x1-x0)||1)*(W-PL-PR);
|
| 113 |
+
const sy = v => PT + (1-(v-y0)/((y1-y0)||1))*(H-PT-PB);
|
| 114 |
+
const d = pts.map((p,i)=> (i?'L':'M')+sx(p.tokens).toFixed(1)+' '+sy(p.loss).toFixed(1)).join(' ');
|
| 115 |
+
const area = d + ` L${sx(x1).toFixed(1)} ${H-PB} L${sx(x0).toFixed(1)} ${H-PB} Z`;
|
| 116 |
+
let g='';
|
| 117 |
+
for(let i=0;i<=TICKS;i++){
|
| 118 |
+
const v = y0+(y1-y0)*i/TICKS, y = sy(v);
|
| 119 |
+
g += `<line class="gl" x1="${PL}" y1="${y.toFixed(1)}" x2="${W-PR}" y2="${y.toFixed(1)}"/>`;
|
| 120 |
+
g += `<text class="ax" x="${PL-7}" y="${(y+3.5).toFixed(1)}" text-anchor="end">${v.toFixed(2)}</text>`;
|
| 121 |
+
}
|
| 122 |
+
for(let i=0;i<=TICKS;i++){
|
| 123 |
+
const v = x0+(x1-x0)*i/TICKS, x = sx(v);
|
| 124 |
+
const anchor = i===0 ? 'start' : (i===TICKS ? 'end' : 'middle');
|
| 125 |
+
g += `<text class="ax" x="${x.toFixed(1)}" y="${H-PB+18}" text-anchor="${anchor}">${NX.fmtTokens(v)}</text>`;
|
| 126 |
+
}
|
| 127 |
+
const last = pts[pts.length-1];
|
| 128 |
+
el.innerHTML = `<defs><linearGradient id="lg" x1="0" y1="0" x2="0" y2="1">
|
| 129 |
+
<stop offset="0%" stop-color="#2ee6a8" stop-opacity=".22"/>
|
| 130 |
+
<stop offset="100%" stop-color="#2ee6a8" stop-opacity="0"/>
|
| 131 |
+
</linearGradient></defs>
|
| 132 |
+
${g}
|
| 133 |
+
<path class="ar" d="${area}"/>
|
| 134 |
+
<path class="ln" d="${d}"/>
|
| 135 |
+
<circle cx="${sx(last.tokens).toFixed(1)}" cy="${sy(last.loss).toFixed(1)}"
|
| 136 |
+
r="4" fill="#2ee6a8"/>
|
| 137 |
+
<text class="ax" x="${W-PR}" y="${PT+2}" text-anchor="end"
|
| 138 |
+
style="fill:#2ee6a8;font-size:12px">loss ${last.loss.toFixed(4)}</text>`;
|
| 139 |
+
},
|
| 140 |
+
|
| 141 |
+
poll(fn, ms){
|
| 142 |
+
let stop=false;
|
| 143 |
+
const tick = async () => {
|
| 144 |
+
if(stop) return;
|
| 145 |
+
try { await fn(); } catch(e){ /* keep polling */ }
|
| 146 |
+
if(!stop) setTimeout(tick, ms);
|
| 147 |
+
};
|
| 148 |
+
tick();
|
| 149 |
+
return ()=>{ stop=true; };
|
| 150 |
+
}
|
| 151 |
+
};
|
| 152 |
+
window.NX = NX;
|
| 153 |
+
|
| 154 |
+
/* Charts use pixel-sized viewBoxes, so they must be redrawn when the box
|
| 155 |
+
changes — rotating a phone, or opening the keyboard. */
|
| 156 |
+
let _rsT;
|
| 157 |
+
addEventListener('resize', () => {
|
| 158 |
+
clearTimeout(_rsT);
|
| 159 |
+
_rsT = setTimeout(() => {
|
| 160 |
+
document.querySelectorAll('svg.chart').forEach(el => {
|
| 161 |
+
if(el._pts) NX.lossChart(el, el._pts);
|
| 162 |
+
});
|
| 163 |
+
}, 160);
|
| 164 |
+
});
|
web/templates/about.html
ADDED
|
@@ -0,0 +1,170 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{% extends "base.html" %}
|
| 2 |
+
{% set nav = 'about' %}
|
| 3 |
+
{% block title %}How it works — NanoDex{% endblock %}
|
| 4 |
+
{% block body %}
|
| 5 |
+
|
| 6 |
+
<div style="max-width:760px">
|
| 7 |
+
<span class="eyebrow">How it works</span>
|
| 8 |
+
<h1 style="font-size:30px;margin:6px 0 10px">What actually happens when you
|
| 9 |
+
press the button</h1>
|
| 10 |
+
<p class="sub" style="font-size:16px">No fine-tuning, no adapters, no
|
| 11 |
+
distillation. A transformer is created with random weights and taught to
|
| 12 |
+
predict the next token, from zero.</p>
|
| 13 |
+
</div>
|
| 14 |
+
|
| 15 |
+
<section style="margin-top:36px">
|
| 16 |
+
<div class="card pad-lg">
|
| 17 |
+
<div class="step-n">1</div>
|
| 18 |
+
<h2 style="font-size:19px">Architecture</h2>
|
| 19 |
+
<p class="sub" style="margin-top:8px">A standard modern decoder-only
|
| 20 |
+
transformer — <span class="mono">LlamaForCausalLM</span>: SiLU MLPs, RMSNorm
|
| 21 |
+
(ε=1e-5), rotary position embeddings (θ=10000), grouped-query attention,
|
| 22 |
+
tied input/output embeddings, no biases. The four tiers are that same
|
| 23 |
+
recipe, narrowed and shortened until it fits the parameter budget.</p>
|
| 24 |
+
|
| 25 |
+
<div style="overflow-x:auto;margin-top:18px">
|
| 26 |
+
<table>
|
| 27 |
+
<thead><tr><th>Tier</th><th>Parameters</th><th>Layers</th><th>Hidden</th>
|
| 28 |
+
<th>Heads (KV)</th><th>FFN</th><th>Tokens / step</th></tr></thead>
|
| 29 |
+
<tbody>
|
| 30 |
+
{% for t in tiers %}
|
| 31 |
+
<tr><td><b>{{ t.label }}</b></td>
|
| 32 |
+
<td class="mono">{{ "{:,}".format(t.params) }}</td>
|
| 33 |
+
<td>{{ t.layers }}</td><td>{{ t.hidden }}</td>
|
| 34 |
+
<td>{{ t.heads }} ({{ t.kv_heads }})</td><td>{{ t.ffn }}</td>
|
| 35 |
+
<td class="mono">{{ "{:,}".format(t.batch_tokens) }}</td></tr>
|
| 36 |
+
{% endfor %}
|
| 37 |
+
</tbody>
|
| 38 |
+
</table>
|
| 39 |
+
</div>
|
| 40 |
+
</div>
|
| 41 |
+
</section>
|
| 42 |
+
|
| 43 |
+
<section style="margin-top:16px">
|
| 44 |
+
<div class="card pad-lg">
|
| 45 |
+
<div class="step-n">2</div>
|
| 46 |
+
<h2 style="font-size:19px">Tokenizer</h2>
|
| 47 |
+
<p class="sub" style="margin-top:8px">A byte-level BPE with
|
| 48 |
+
<b>{{ "{:,}".format(vocab) }} tokens</b>, trained on fineweb-edu itself
|
| 49 |
+
(~2.7 characters per token). This matters more than it sounds: a typical
|
| 50 |
+
small-LM vocabulary of ~49k tokens would cost 28M embedding parameters on
|
| 51 |
+
its own — more than three times the largest model here. A lean vocabulary is
|
| 52 |
+
what makes "a 500k-parameter language model" a sentence that means something. The
|
| 53 |
+
parameter counts above are real totals, embeddings included.</p>
|
| 54 |
+
<p class="sub" style="margin-top:12px">The vocabulary is deliberately the
|
| 55 |
+
same for every tier, and stays at 2,048 even for the 8M model. Two reasons:
|
| 56 |
+
at a 500k total budget anything larger would be almost entirely
|
| 57 |
+
embeddings, and a shared vocabulary is the only thing that makes the
|
| 58 |
+
losses on the <a href="/explore" style="color:var(--accent)">leaderboard</a>
|
| 59 |
+
comparable at all — change the tokenizer and cross-entropy stops meaning
|
| 60 |
+
the same thing between two models.</p>
|
| 61 |
+
</div>
|
| 62 |
+
</section>
|
| 63 |
+
|
| 64 |
+
<section style="margin-top:16px">
|
| 65 |
+
<div class="card pad-lg">
|
| 66 |
+
<div class="step-n">3</div>
|
| 67 |
+
<h2 style="font-size:19px">Data</h2>
|
| 68 |
+
<p class="sub" style="margin-top:8px">
|
| 69 |
+
<a href="https://huggingface.co/datasets/HuggingFaceFW/fineweb-edu"
|
| 70 |
+
target="_blank" rel="noopener" style="color:var(--accent)">HuggingFaceFW/fineweb-edu</a>,
|
| 71 |
+
config <span class="mono">sample-10BT</span> — filtered educational web text.
|
| 72 |
+
Instead of every run re-streaming and re-tokenizing the corpus, this site
|
| 73 |
+
tokenizes a {{ "%.2fB"|format(cache_target / 1000000000) }}-token slice
|
| 74 |
+
<b>once</b> into a flat <span class="mono">uint16</span> file
|
| 75 |
+
(~{{ "%.1f"|format(cache_target * 2 / 1024**3) }} GB) that all runs read
|
| 76 |
+
from, so the workers are never waiting on data. Building it is a one-time
|
| 77 |
+
cost per container; a run that asks for more tokens than are cached yet
|
| 78 |
+
waits for the writer to catch up, and says so in its log. Every run starts at
|
| 79 |
+
its own random offset, so two identical configurations don't see identical
|
| 80 |
+
token order.</p>
|
| 81 |
+
</div>
|
| 82 |
+
</section>
|
| 83 |
+
|
| 84 |
+
<section style="margin-top:16px">
|
| 85 |
+
<div class="card pad-lg">
|
| 86 |
+
<div class="step-n">4</div>
|
| 87 |
+
<h2 style="font-size:19px">Training</h2>
|
| 88 |
+
<div class="grid g2" style="margin-top:14px">
|
| 89 |
+
<div>
|
| 90 |
+
<div class="kv"><span>Optimizer</span><b>AdamW β=(0.9, 0.95)</b></div>
|
| 91 |
+
<div class="kv"><span>Weight decay</span><b>0.1</b></div>
|
| 92 |
+
<div class="kv"><span>Gradient clipping</span><b>1.0</b></div>
|
| 93 |
+
<div class="kv"><span>Schedule</span><b>2% warmup → cosine to 10%</b></div>
|
| 94 |
+
</div>
|
| 95 |
+
<div>
|
| 96 |
+
<div class="kv"><span>Precision</span><b>bf16 autocast on GPU</b></div>
|
| 97 |
+
<div class="kv"><span>Context length</span><b>{{ seq_len }} tokens</b></div>
|
| 98 |
+
<div class="kv"><span>Token budget</span>
|
| 99 |
+
<b>{{ min_tokens // 10**6 }}M – 1.5B</b></div>
|
| 100 |
+
<div class="kv"><span>Concurrent runs</span>
|
| 101 |
+
<b>{{ max_active }} per person</b></div>
|
| 102 |
+
</div>
|
| 103 |
+
</div>
|
| 104 |
+
</div>
|
| 105 |
+
</section>
|
| 106 |
+
|
| 107 |
+
<section style="margin-top:16px">
|
| 108 |
+
<div class="card pad-lg">
|
| 109 |
+
<div class="step-n">5</div>
|
| 110 |
+
<h2 style="font-size:19px">Queue</h2>
|
| 111 |
+
<p class="sub" style="margin-top:8px">One worker thread per available device
|
| 112 |
+
claims the oldest queued run atomically, so two workers can never grab the
|
| 113 |
+
same job. This deployment currently has
|
| 114 |
+
<b>{{ n_devices }} {{ 'GPU' if on_gpu else 'CPU' }} worker{{ 's' if n_devices != 1 }}</b>,
|
| 115 |
+
which means {{ n_devices }} run{{ 's' if n_devices != 1 }} can train at
|
| 116 |
+
once. Estimated times shown in the wizard start from a prior and then
|
| 117 |
+
recalibrate from the median throughput of runs that actually finished here.</p>
|
| 118 |
+
</div>
|
| 119 |
+
</section>
|
| 120 |
+
|
| 121 |
+
<section style="margin-top:16px">
|
| 122 |
+
<div class="card pad-lg">
|
| 123 |
+
<div class="step-n">6</div>
|
| 124 |
+
<h2 style="font-size:19px">Publishing</h2>
|
| 125 |
+
<p class="sub" style="margin-top:8px">When a run finishes you can push it to
|
| 126 |
+
your own Hugging Face namespace with the OAuth token from your session:
|
| 127 |
+
<span class="mono">config.json</span>,
|
| 128 |
+
<span class="mono">model.safetensors</span>, tokenizer files,
|
| 129 |
+
<span class="mono">generation_config.json</span>, a
|
| 130 |
+
<span class="mono">training_run.json</span> recording the complete recipe,
|
| 131 |
+
and a generated model card. It loads with
|
| 132 |
+
<span class="mono">transformers</span> like any other model on the Hub.</p>
|
| 133 |
+
</div>
|
| 134 |
+
</section>
|
| 135 |
+
|
| 136 |
+
<section style="margin-top:16px">
|
| 137 |
+
<div class="card pad-lg">
|
| 138 |
+
<div class="step-n">7</div>
|
| 139 |
+
<h2 style="font-size:19px">Durable storage</h2>
|
| 140 |
+
<p class="sub" style="margin-top:8px">A Space's filesystem is wiped on every
|
| 141 |
+
rebuild and restart, so finished runs, their loss curves and the token
|
| 142 |
+
cache are mirrored to a Hugging Face bucket. The archive is append-only
|
| 143 |
+
and keyed by run id — a finished run is written once and never mutated,
|
| 144 |
+
which is why a rolling deploy running two containers at once cannot
|
| 145 |
+
corrupt it. History comes back on boot; weights are pulled on demand the
|
| 146 |
+
first time somebody opens the playground.</p>
|
| 147 |
+
<p class="sub" style="margin-top:12px">Runs that are mid-training when a
|
| 148 |
+
container dies are <b>not</b> recoverable — the optimizer state only ever
|
| 149 |
+
exists in that process. They come back marked interrupted.</p>
|
| 150 |
+
</div>
|
| 151 |
+
</section>
|
| 152 |
+
|
| 153 |
+
<section style="margin-top:36px">
|
| 154 |
+
<div class="card pad-lg" style="border-color:var(--line-2)">
|
| 155 |
+
<span class="eyebrow">A note on scale</span>
|
| 156 |
+
<h2 style="font-size:20px;margin:8px 0 10px">Deliberately over-trained</h2>
|
| 157 |
+
<p class="sub">1.5 billion tokens into an 8M-parameter model is roughly 9×
|
| 158 |
+
past the Chinchilla compute-optimal ratio of 20 tokens per parameter —
|
| 159 |
+
and into a 500k-parameter model, over 150× past it. That's intentional.
|
| 160 |
+
Tiny models keep improving long after the
|
| 161 |
+
compute-optimal point, and the goal here was never FLOP efficiency — it's
|
| 162 |
+
watching cross-entropy fall from <b>7.6</b> (uniform noise over
|
| 163 |
+
{{ "{:,}".format(vocab) }} tokens) toward <b>4</b>, and seeing a network
|
| 164 |
+
that started as pure randomness begin to emit English words.</p>
|
| 165 |
+
<p class="sub" style="margin-top:12px">Nothing trained here is a useful
|
| 166 |
+
assistant, and nothing it says is factual. That was never the point.</p>
|
| 167 |
+
</div>
|
| 168 |
+
</section>
|
| 169 |
+
|
| 170 |
+
{% endblock %}
|
web/templates/base.html
ADDED
|
@@ -0,0 +1,66 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
<!doctype html>
|
| 2 |
+
<html lang="en">
|
| 3 |
+
<head>
|
| 4 |
+
<meta charset="utf-8">
|
| 5 |
+
<meta name="viewport" content="width=device-width,initial-scale=1">
|
| 6 |
+
<title>{% block title %}NanoDex{% endblock %}</title>
|
| 7 |
+
<meta name="description" content="Pre-train a small decoder-only language model from scratch, in the cloud.">
|
| 8 |
+
<link rel="stylesheet" href="/static/app.css">
|
| 9 |
+
<link rel="icon" href="data:image/svg+xml,<svg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 100 100'><text y='.9em' font-size='90'>🧬</text></svg>">
|
| 10 |
+
{% block head %}{% endblock %}
|
| 11 |
+
</head>
|
| 12 |
+
<body>
|
| 13 |
+
|
| 14 |
+
<nav>
|
| 15 |
+
<div class="wrap">
|
| 16 |
+
<a class="brand" href="/"><span class="dot"></span>NanoDex</a>
|
| 17 |
+
<div class="navlinks">
|
| 18 |
+
{% if user %}
|
| 19 |
+
<a href="/" class="{{ 'on' if nav=='home' }}">Home</a>
|
| 20 |
+
<a href="/models" class="{{ 'on' if nav=='models' }}">My models</a>
|
| 21 |
+
<a href="/train" class="{{ 'on' if nav=='train' }}">New run</a>
|
| 22 |
+
{% endif %}
|
| 23 |
+
<a href="/explore" class="{{ 'on' if nav=='explore' }}">Explore</a>
|
| 24 |
+
<a href="/queue" class="{{ 'on' if nav=='queue' }}">Queue</a>
|
| 25 |
+
<a href="/about" class="{{ 'on' if nav=='about' }}">How it works</a>
|
| 26 |
+
</div>
|
| 27 |
+
{% if user %}
|
| 28 |
+
<a class="usermenu" href="/profile">
|
| 29 |
+
{% if user.picture %}<img class="avatar" src="{{ user.picture }}" alt="">
|
| 30 |
+
{% else %}<span class="avatar"></span>{% endif %}
|
| 31 |
+
<span>{{ user.username }}</span>
|
| 32 |
+
</a>
|
| 33 |
+
{% else %}
|
| 34 |
+
<a class="btn primary sm" target="_top" href="/login"><span class="signin-long">Sign in with Hugging Face</span><span class="signin-short">Sign in</span></a>
|
| 35 |
+
{% endif %}
|
| 36 |
+
</div>
|
| 37 |
+
</nav>
|
| 38 |
+
|
| 39 |
+
<main><div class="wrap">{% block body %}{% endblock %}</div></main>
|
| 40 |
+
|
| 41 |
+
<footer><div class="wrap">
|
| 42 |
+
<div>NanoDex · train a decoder-only language model from scratch</div>
|
| 43 |
+
<div>
|
| 44 |
+
<a href="/about">How it works</a> ·
|
| 45 |
+
<a href="https://huggingface.co/datasets/HuggingFaceFW/fineweb-edu" target="_blank" rel="noopener">fineweb-edu</a> ·
|
| 46 |
+
<a href="https://huggingface.co/hugging-science" target="_blank" rel="noopener">hugging-science</a>
|
| 47 |
+
</div>
|
| 48 |
+
</div></footer>
|
| 49 |
+
|
| 50 |
+
<script>
|
| 51 |
+
/* huggingface.co refuses to be displayed in a frame, so the OAuth handshake
|
| 52 |
+
must run as a top-level navigation — inside the Space iframe it would just
|
| 53 |
+
render a blank grey box. Retarget anything we missed, and show a hint. */
|
| 54 |
+
(function(){
|
| 55 |
+
if (window.self === window.top) return;
|
| 56 |
+
document.documentElement.classList.add('framed');
|
| 57 |
+
addEventListener('DOMContentLoaded', function(){
|
| 58 |
+
document.querySelectorAll('a[href^="/login"], a[href^="/logout"]')
|
| 59 |
+
.forEach(function(a){ a.target = '_top'; });
|
| 60 |
+
});
|
| 61 |
+
})();
|
| 62 |
+
</script>
|
| 63 |
+
<script src="/static/app.js"></script>
|
| 64 |
+
{% block scripts %}{% endblock %}
|
| 65 |
+
</body>
|
| 66 |
+
</html>
|
web/templates/detail.html
ADDED
|
@@ -0,0 +1,319 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{% extends "base.html" %}
|
| 2 |
+
{% set nav = 'models' %}
|
| 3 |
+
{% block title %}{{ job.name }} — NanoDex{% endblock %}
|
| 4 |
+
{% block body %}
|
| 5 |
+
|
| 6 |
+
<a href="{{ '/models' if owner else '/explore' }}" class="btn ghost sm"
|
| 7 |
+
style="margin-bottom:16px">← {{ 'My models' if owner else 'Explore' }}</a>
|
| 8 |
+
|
| 9 |
+
<div class="sect-h" style="align-items:flex-start;margin-bottom:20px">
|
| 10 |
+
<div>
|
| 11 |
+
<div style="display:flex;align-items:center;gap:11px;flex-wrap:wrap">
|
| 12 |
+
<h1 style="font-size:27px">{{ job.name }}</h1>
|
| 13 |
+
<span id="statusChip"></span>
|
| 14 |
+
</div>
|
| 15 |
+
<p class="sub" style="margin-top:5px">
|
| 16 |
+
{{ job.tier_label }} · {{ "{:,}".format(job.n_params) }} parameters ·
|
| 17 |
+
by <a href="/u/{{ job.username }}" style="color:var(--accent)">@{{ job.username }}</a>
|
| 18 |
+
</p>
|
| 19 |
+
</div>
|
| 20 |
+
{% if owner %}
|
| 21 |
+
<div style="display:flex;gap:8px">
|
| 22 |
+
<button class="btn danger sm" id="cancelBtn" style="display:none">Cancel run</button>
|
| 23 |
+
</div>
|
| 24 |
+
{% endif %}
|
| 25 |
+
</div>
|
| 26 |
+
|
| 27 |
+
<div id="banner"></div>
|
| 28 |
+
|
| 29 |
+
<div class="card" style="margin-bottom:16px">
|
| 30 |
+
<div style="display:flex;justify-content:space-between;align-items:baseline;margin-bottom:10px">
|
| 31 |
+
<span class="eyebrow">Progress</span>
|
| 32 |
+
<span class="mono" id="pctLabel" style="color:var(--accent);font-weight:640">—</span>
|
| 33 |
+
</div>
|
| 34 |
+
<div class="bar"><i id="progBar" style="width:0%"></i></div>
|
| 35 |
+
<div class="grid g4" style="margin-top:18px;gap:14px">
|
| 36 |
+
<div><span class="eyebrow">Tokens seen</span>
|
| 37 |
+
<div class="mono" style="font-size:20px;font-weight:650" id="m-tokens">—</div>
|
| 38 |
+
<small class="muted" id="m-target"> </small></div>
|
| 39 |
+
<div><span class="eyebrow">Step</span>
|
| 40 |
+
<div class="mono" style="font-size:20px;font-weight:650" id="m-step">—</div>
|
| 41 |
+
<small class="muted" id="m-steps"> </small></div>
|
| 42 |
+
<div><span class="eyebrow">Loss</span>
|
| 43 |
+
<div class="mono" style="font-size:20px;font-weight:650" id="m-loss">—</div>
|
| 44 |
+
<small class="muted" id="m-ppl"> </small></div>
|
| 45 |
+
<div><span class="eyebrow">Throughput</span>
|
| 46 |
+
<div class="mono" style="font-size:20px;font-weight:650" id="m-tps">—</div>
|
| 47 |
+
<small class="muted" id="m-eta"> </small></div>
|
| 48 |
+
</div>
|
| 49 |
+
</div>
|
| 50 |
+
|
| 51 |
+
<div class="card" style="margin-bottom:16px">
|
| 52 |
+
<div class="sect-h" style="margin-bottom:8px">
|
| 53 |
+
<h2 style="font-size:16px">Loss curve</h2>
|
| 54 |
+
<span class="sub">cross-entropy vs tokens seen</span>
|
| 55 |
+
</div>
|
| 56 |
+
<svg class="chart" id="chart" preserveAspectRatio="xMidYMid meet"></svg>
|
| 57 |
+
</div>
|
| 58 |
+
|
| 59 |
+
<div class="grid g2" style="margin-bottom:16px">
|
| 60 |
+
<div class="card">
|
| 61 |
+
<h2 style="font-size:16px;margin-bottom:12px">Architecture</h2>
|
| 62 |
+
<div class="kv"><span>Class</span><b class="mono">LlamaForCausalLM</b></div>
|
| 63 |
+
<div class="kv"><span>Layers</span><b>{{ job.arch.layers }}</b></div>
|
| 64 |
+
<div class="kv"><span>Hidden size</span><b>{{ job.arch.hidden }}</b></div>
|
| 65 |
+
<div class="kv"><span>Attention heads</span><b>{{ job.arch.heads }} ({{ job.arch.kv_heads }} KV, head dim {{ job.arch.head_dim }})</b></div>
|
| 66 |
+
<div class="kv"><span>FFN size</span><b>{{ job.arch.ffn }}</b></div>
|
| 67 |
+
<div class="kv"><span>Context</span><b>{{ job.arch.ctx }} tokens</b></div>
|
| 68 |
+
<div class="kv"><span>Vocabulary</span><b>{{ "{:,}".format(job.arch.vocab) }}</b></div>
|
| 69 |
+
<div class="kv"><span>Parameters</span><b>{{ "{:,}".format(job.n_params) }}</b></div>
|
| 70 |
+
</div>
|
| 71 |
+
<div class="card">
|
| 72 |
+
<h2 style="font-size:16px;margin-bottom:12px">Run</h2>
|
| 73 |
+
<div class="kv"><span>Dataset</span><b>fineweb-edu</b></div>
|
| 74 |
+
<div class="kv"><span>Token budget</span><b id="d-target">—</b></div>
|
| 75 |
+
<div class="kv"><span>Optimizer</span><b>AdamW β=0.9/0.95 wd=0.1</b></div>
|
| 76 |
+
<div class="kv"><span>Schedule</span><b>2% warmup → cosine</b></div>
|
| 77 |
+
<div class="kv"><span>Learning rate</span><b id="d-lr">—</b></div>
|
| 78 |
+
<div class="kv"><span>Best loss</span><b id="d-best">—</b></div>
|
| 79 |
+
<div class="kv"><span>Worker</span><b id="d-gpu">—</b></div>
|
| 80 |
+
<div class="kv"><span>Queued</span><b id="d-created">—</b></div>
|
| 81 |
+
</div>
|
| 82 |
+
</div>
|
| 83 |
+
|
| 84 |
+
{% if owner %}
|
| 85 |
+
<div class="card" id="publishCard" style="display:none;margin-bottom:16px">
|
| 86 |
+
<h2 style="font-size:16px">Publish to your Hugging Face account</h2>
|
| 87 |
+
<p class="sub" style="margin:6px 0 14px">Uploads weights, config, tokenizer, the
|
| 88 |
+
full training recipe and a generated model card to
|
| 89 |
+
<span class="mono">{{ user.username }}/<name></span>.</p>
|
| 90 |
+
<div style="display:flex;gap:10px;flex-wrap:wrap">
|
| 91 |
+
<input type="text" id="repoName" placeholder="{{ job.name }}" style="flex:1;min-width:220px">
|
| 92 |
+
<button class="btn primary" id="publishBtn">Publish</button>
|
| 93 |
+
</div>
|
| 94 |
+
</div>
|
| 95 |
+
{% endif %}
|
| 96 |
+
|
| 97 |
+
<div class="card" id="pgCard" style="display:none">
|
| 98 |
+
<div class="sect-h" style="margin-bottom:6px">
|
| 99 |
+
<h2 style="font-size:16px">Playground</h2>
|
| 100 |
+
<span id="ckptChip"></span>
|
| 101 |
+
</div>
|
| 102 |
+
<p class="sub" style="margin:0 0 14px" id="pgBlurb">Generate text with this
|
| 103 |
+
model. At this scale expect English-shaped output with no facts in it —
|
| 104 |
+
that's the honest result.</p>
|
| 105 |
+
<label class="fl">Prompt</label>
|
| 106 |
+
<textarea id="pgPrompt" rows="2">The mitochondria is</textarea>
|
| 107 |
+
<div class="grid g4" style="margin-top:14px">
|
| 108 |
+
<div><label class="fl">New tokens <span class="mono" id="v-max">80</span></label>
|
| 109 |
+
<input type="range" id="pgMax" min="8" max="256" step="8" value="80"></div>
|
| 110 |
+
<div><label class="fl">Temperature <span class="mono" id="v-temp">0.85</span></label>
|
| 111 |
+
<input type="range" id="pgTemp" min="0.1" max="2" step="0.05" value="0.85"></div>
|
| 112 |
+
<div><label class="fl">Top-k <span class="mono" id="v-topk">40</span></label>
|
| 113 |
+
<input type="range" id="pgTopk" min="0" max="200" step="5" value="40"></div>
|
| 114 |
+
<div><label class="fl">Top-p <span class="mono" id="v-topp">0.95</span></label>
|
| 115 |
+
<input type="range" id="pgTopp" min="0.1" max="1" step="0.05" value="0.95"></div>
|
| 116 |
+
</div>
|
| 117 |
+
<div style="display:flex;gap:10px;align-items:center;margin-top:16px;flex-wrap:wrap">
|
| 118 |
+
{% if user %}<button class="btn primary" id="pgGo">Generate</button>
|
| 119 |
+
{% else %}<a class="btn primary" target="_top"
|
| 120 |
+
href="/login?next=/models/{{ job.id }}">Sign in to generate</a>{% endif %}
|
| 121 |
+
{% if user %}
|
| 122 |
+
<label class="sub" style="display:flex;align-items:center;gap:7px;font-size:13px">
|
| 123 |
+
<input type="checkbox" id="pgAuto" style="width:auto;accent-color:var(--accent)">
|
| 124 |
+
re-run on every new checkpoint
|
| 125 |
+
</label>
|
| 126 |
+
{% endif %}
|
| 127 |
+
<span class="mono muted" id="pgStats" style="font-size:12px"></span>
|
| 128 |
+
</div>
|
| 129 |
+
<div id="pgOut" class="pgout" style="margin-top:14px;display:none"></div>
|
| 130 |
+
</div>
|
| 131 |
+
|
| 132 |
+
<div class="card" style="margin-top:16px">
|
| 133 |
+
<h2 style="font-size:16px;margin-bottom:12px">Logs</h2>
|
| 134 |
+
<div class="logs" id="logs">waiting…</div>
|
| 135 |
+
</div>
|
| 136 |
+
|
| 137 |
+
{% endblock %}
|
| 138 |
+
{% block scripts %}
|
| 139 |
+
<script>
|
| 140 |
+
const JOB_ID = {{ job.id | tojson }};
|
| 141 |
+
const OWNER = {{ owner | tojson }};
|
| 142 |
+
const SIGNED_IN = {{ (user is not none) | tojson }};
|
| 143 |
+
const INFER_DEVICE = {{ infer_device | tojson }};
|
| 144 |
+
let stopPoll = null;
|
| 145 |
+
let lastCkpt = null;
|
| 146 |
+
|
| 147 |
+
function banner(j){
|
| 148 |
+
const el = document.getElementById('banner');
|
| 149 |
+
if(j.status === 'queued'){
|
| 150 |
+
el.innerHTML = `<div class="card" style="margin-bottom:16px;border-color:#3b2f63">
|
| 151 |
+
<b>Position #${j.queue_position || '?'} in the queue.</b>
|
| 152 |
+
<span class="sub"> A worker will pick it up automatically — you can close this tab.</span>
|
| 153 |
+
</div>`;
|
| 154 |
+
} else if(j.status === 'failed'){
|
| 155 |
+
el.innerHTML = `<div class="card" style="margin-bottom:16px;border-color:#4c1d24">
|
| 156 |
+
<b style="color:var(--red)">This run failed.</b>
|
| 157 |
+
<div class="mono sub" style="margin-top:6px">${NX.esc(j.error||'')}</div></div>`;
|
| 158 |
+
} else if(j.status === 'done' && j.pushed_repo){
|
| 159 |
+
el.innerHTML = `<div class="card" style="margin-bottom:16px;border-color:var(--accent-2)">
|
| 160 |
+
<b>Published →</b> <a class="mono" style="color:var(--accent)"
|
| 161 |
+
href="https://huggingface.co/${NX.esc(j.pushed_repo)}" target="_blank"
|
| 162 |
+
rel="noopener">${NX.esc(j.pushed_repo)}</a></div>`;
|
| 163 |
+
} else if(j.status === 'cancelled'){
|
| 164 |
+
el.innerHTML = `<div class="card" style="margin-bottom:16px">
|
| 165 |
+
<b>Cancelled.</b> <span class="sub">Stopped before finishing.</span></div>`;
|
| 166 |
+
} else { el.innerHTML = ''; }
|
| 167 |
+
}
|
| 168 |
+
|
| 169 |
+
async function refresh(){
|
| 170 |
+
const j = await NX.get('/api/jobs/' + JOB_ID);
|
| 171 |
+
const pct = (j.progress * 100);
|
| 172 |
+
|
| 173 |
+
document.getElementById('statusChip').innerHTML = NX.chip(j.status);
|
| 174 |
+
document.getElementById('pctLabel').textContent = pct.toFixed(1) + '%';
|
| 175 |
+
document.getElementById('progBar').style.width = pct.toFixed(2) + '%';
|
| 176 |
+
document.getElementById('m-tokens').textContent = NX.fmtTokens(j.tokens_seen);
|
| 177 |
+
document.getElementById('m-target').textContent = 'of ' + NX.fmtTokens(j.target_tokens);
|
| 178 |
+
document.getElementById('m-step').textContent = Number(j.step).toLocaleString();
|
| 179 |
+
document.getElementById('m-steps').textContent = 'of ' + Number(j.total_steps).toLocaleString();
|
| 180 |
+
document.getElementById('m-loss').textContent = j.loss ? j.loss.toFixed(4) : '—';
|
| 181 |
+
document.getElementById('m-ppl').textContent = j.loss
|
| 182 |
+
? 'ppl ' + Math.exp(Math.min(j.loss,20)).toFixed(1) : 'not started';
|
| 183 |
+
document.getElementById('m-tps').textContent = j.tok_per_s
|
| 184 |
+
? Math.round(j.tok_per_s/1e3) + 'k/s' : '—';
|
| 185 |
+
document.getElementById('m-eta').textContent = (j.status==='running' && j.eta_s)
|
| 186 |
+
? NX.fmtTime(j.eta_s) + ' left' : (j.status==='done' ? 'completed' : '');
|
| 187 |
+
|
| 188 |
+
document.getElementById('d-target').textContent = NX.fmtTokens(j.target_tokens) + ' tokens';
|
| 189 |
+
document.getElementById('d-lr').textContent = j.lr ? j.lr.toExponential(2) : '—';
|
| 190 |
+
document.getElementById('d-best').textContent = j.best_loss ? j.best_loss.toFixed(4) : '—';
|
| 191 |
+
document.getElementById('d-gpu').textContent =
|
| 192 |
+
j.gpu_index === null || j.gpu_index === undefined ? 'not assigned' : ('worker ' + j.gpu_index);
|
| 193 |
+
document.getElementById('d-created').textContent = NX.ago(j.created_at);
|
| 194 |
+
|
| 195 |
+
banner(j);
|
| 196 |
+
|
| 197 |
+
if(OWNER){
|
| 198 |
+
const cb = document.getElementById('cancelBtn');
|
| 199 |
+
if(cb) cb.style.display =
|
| 200 |
+
(j.status==='queued' || j.status==='running') ? 'inline-flex' : 'none';
|
| 201 |
+
const pc = document.getElementById('publishCard');
|
| 202 |
+
if(pc) pc.style.display = (j.status==='done' && !j.pushed_repo) ? 'block' : 'none';
|
| 203 |
+
}
|
| 204 |
+
|
| 205 |
+
{
|
| 206 |
+
// The playground is available to anyone signed in, from the moment the
|
| 207 |
+
// first live checkpoint lands — so you can hear someone else's model
|
| 208 |
+
// mid-training, not only your own at the end.
|
| 209 |
+
const pg = document.getElementById('pgCard');
|
| 210 |
+
if(pg) pg.style.display = j.has_checkpoint ? 'block' : 'none';
|
| 211 |
+
|
| 212 |
+
const chip = document.getElementById('ckptChip');
|
| 213 |
+
const blurb = document.getElementById('pgBlurb');
|
| 214 |
+
if(chip){
|
| 215 |
+
if(j.status === 'done'){
|
| 216 |
+
chip.innerHTML = `<span class="chip done"><i></i>final weights</span>`;
|
| 217 |
+
if(blurb) blurb.textContent = OWNER
|
| 218 |
+
? "Generate text with the model you trained. At this scale expect "
|
| 219 |
+
+ "English-shaped output with no facts in it — that's the honest result."
|
| 220 |
+
: "Generate text with this finished model. At this scale expect "
|
| 221 |
+
+ "English-shaped output with no facts in it — that's the honest result.";
|
| 222 |
+
} else if(j.live_step !== null && j.live_step !== undefined){
|
| 223 |
+
chip.innerHTML = `<span class="chip running"><i></i>live checkpoint ·
|
| 224 |
+
step ${Number(j.live_step).toLocaleString()}</span>`;
|
| 225 |
+
if(blurb) blurb.textContent =
|
| 226 |
+
`Weights as of step ${Number(j.live_step).toLocaleString()} of `
|
| 227 |
+
+ `${Number(j.total_steps).toLocaleString()}, loaded on ${INFER_DEVICE}. `
|
| 228 |
+
+ "It gets less incoherent as the run goes on — generate again later "
|
| 229 |
+
+ "and compare.";
|
| 230 |
+
} else {
|
| 231 |
+
chip.innerHTML = '';
|
| 232 |
+
}
|
| 233 |
+
}
|
| 234 |
+
|
| 235 |
+
// auto re-run when a fresher snapshot appears
|
| 236 |
+
const auto = SIGNED_IN ? document.getElementById('pgAuto') : null;
|
| 237 |
+
if(auto && auto.checked && j.live_step !== lastCkpt && j.live_step != null){
|
| 238 |
+
lastCkpt = j.live_step;
|
| 239 |
+
generate(true);
|
| 240 |
+
} else if(j.live_step != null && lastCkpt === null){
|
| 241 |
+
lastCkpt = j.live_step;
|
| 242 |
+
}
|
| 243 |
+
}
|
| 244 |
+
|
| 245 |
+
const {metrics} = await NX.get(`/api/jobs/${JOB_ID}/metrics`);
|
| 246 |
+
NX.lossChart(document.getElementById('chart'), metrics);
|
| 247 |
+
|
| 248 |
+
const {logs} = await NX.get(`/api/jobs/${JOB_ID}/logs`);
|
| 249 |
+
const box = document.getElementById('logs');
|
| 250 |
+
const atBottom = box.scrollHeight - box.scrollTop - box.clientHeight < 40;
|
| 251 |
+
box.innerHTML = logs.length ? logs.map(l =>
|
| 252 |
+
`<span class="t">${new Date(l.t*1000).toLocaleTimeString()}</span>${NX.esc(l.line)}`
|
| 253 |
+
).join('\n') : 'no logs yet';
|
| 254 |
+
if(atBottom) box.scrollTop = box.scrollHeight;
|
| 255 |
+
|
| 256 |
+
if(['done','failed','cancelled'].includes(j.status) && stopPoll){ stopPoll(); stopPoll = null; }
|
| 257 |
+
}
|
| 258 |
+
|
| 259 |
+
stopPoll = NX.poll(refresh, 3000);
|
| 260 |
+
|
| 261 |
+
const cb = document.getElementById('cancelBtn');
|
| 262 |
+
if(cb) cb.addEventListener('click', async () => {
|
| 263 |
+
if(!confirm('Stop this run? Progress is lost.')) return;
|
| 264 |
+
cb.disabled = true;
|
| 265 |
+
try{ await NX.post(`/api/jobs/${JOB_ID}/cancel`); NX.toast('Cancellation requested.','ok'); }
|
| 266 |
+
catch(e){ NX.toast(NX.esc(e.message),'err'); cb.disabled = false; }
|
| 267 |
+
});
|
| 268 |
+
|
| 269 |
+
const pb = document.getElementById('publishBtn');
|
| 270 |
+
if(pb) pb.addEventListener('click', async () => {
|
| 271 |
+
pb.disabled = true; pb.textContent = 'Uploading…';
|
| 272 |
+
try{
|
| 273 |
+
const r = await NX.post(`/api/jobs/${JOB_ID}/publish`,
|
| 274 |
+
{repo: document.getElementById('repoName').value});
|
| 275 |
+
NX.toast(`Published → <a href="${r.url}" target="_blank"
|
| 276 |
+
style="color:var(--accent)">${NX.esc(r.repo)}</a>`, 'ok');
|
| 277 |
+
if(!stopPoll) refresh();
|
| 278 |
+
}catch(e){ NX.toast(NX.esc(e.message),'err'); }
|
| 279 |
+
finally{ pb.disabled = false; pb.textContent = 'Publish'; }
|
| 280 |
+
});
|
| 281 |
+
|
| 282 |
+
['Max','Temp','Topk','Topp'].forEach(k => {
|
| 283 |
+
const el = document.getElementById('pg'+k);
|
| 284 |
+
if(!el) return;
|
| 285 |
+
const out = document.getElementById('v-'+k.toLowerCase());
|
| 286 |
+
el.addEventListener('input', () => out.textContent = el.value);
|
| 287 |
+
});
|
| 288 |
+
|
| 289 |
+
const pgGo = document.getElementById('pgGo');
|
| 290 |
+
|
| 291 |
+
async function generate(quiet){
|
| 292 |
+
if(!pgGo || pgGo.disabled) return;
|
| 293 |
+
pgGo.disabled = true;
|
| 294 |
+
const label = pgGo.textContent;
|
| 295 |
+
pgGo.textContent = 'Generating…';
|
| 296 |
+
const out = document.getElementById('pgOut');
|
| 297 |
+
try{
|
| 298 |
+
const r = await NX.post(`/api/jobs/${JOB_ID}/generate`, {
|
| 299 |
+
prompt: document.getElementById('pgPrompt').value,
|
| 300 |
+
max_new_tokens: Number(document.getElementById('pgMax').value),
|
| 301 |
+
temperature: Number(document.getElementById('pgTemp').value),
|
| 302 |
+
top_k: Number(document.getElementById('pgTopk').value),
|
| 303 |
+
top_p: Number(document.getElementById('pgTopp').value),
|
| 304 |
+
});
|
| 305 |
+
out.style.display = 'block';
|
| 306 |
+
// The prompt is dimmed so it reads as the seed, not as model output.
|
| 307 |
+
out.innerHTML = `<span class="pgp">${NX.esc(r.prompt)}</span>`
|
| 308 |
+
+ `<span class="pgc">${NX.esc(r.continuation)}</span>`;
|
| 309 |
+
document.getElementById('pgStats').textContent =
|
| 310 |
+
`${r.new_tokens} tokens · ${r.ms} ms · ${r.device} · `
|
| 311 |
+
+ (r.checkpoint === 'final' ? 'final weights'
|
| 312 |
+
: `step ${Number(r.step).toLocaleString()}`);
|
| 313 |
+
}catch(e){ if(!quiet) NX.toast(NX.esc(e.message),'err'); }
|
| 314 |
+
finally{ pgGo.disabled = false; pgGo.textContent = label; }
|
| 315 |
+
}
|
| 316 |
+
|
| 317 |
+
if(pgGo) pgGo.addEventListener('click', () => generate(false));
|
| 318 |
+
</script>
|
| 319 |
+
{% endblock %}
|
web/templates/error.html
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{% extends "base.html" %}
|
| 2 |
+
{% block title %}{{ code }} — NanoDex{% endblock %}
|
| 3 |
+
{% block body %}
|
| 4 |
+
<div style="padding:80px 0;text-align:center">
|
| 5 |
+
<div class="mono" style="font-weight:700;color:var(--line-2);
|
| 6 |
+
letter-spacing:-.04em;line-height:1.1;
|
| 7 |
+
font-size:{{ '56px' if code|string|length <= 4 else '28px' }}">{{ code }}</div>
|
| 8 |
+
<p class="sub" style="margin:12px 0 26px;font-size:16px">{{ detail }}</p>
|
| 9 |
+
<a class="btn primary" href="/">Back home</a>
|
| 10 |
+
</div>
|
| 11 |
+
{% endblock %}
|
web/templates/explore.html
ADDED
|
@@ -0,0 +1,147 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{% extends "base.html" %}
|
| 2 |
+
{% set nav = 'explore' %}
|
| 3 |
+
{% block title %}Explore — NanoDex{% endblock %}
|
| 4 |
+
{% block body %}
|
| 5 |
+
|
| 6 |
+
<div class="sect-h" style="margin-bottom:22px">
|
| 7 |
+
<div>
|
| 8 |
+
<span class="eyebrow">The shelf</span>
|
| 9 |
+
<h1 style="font-size:28px;margin-top:5px">Every model trained here</h1>
|
| 10 |
+
<p class="sub" style="margin-top:5px">Same tokenizer, same corpus, same
|
| 11 |
+
context length for everyone — so the losses below compare directly. Open
|
| 12 |
+
any model and generate text from it.</p>
|
| 13 |
+
</div>
|
| 14 |
+
</div>
|
| 15 |
+
|
| 16 |
+
<div class="grid g4" style="margin-bottom:24px">
|
| 17 |
+
<div class="stat"><span>Models</span><b id="s-done">—</b><small>finished runs</small></div>
|
| 18 |
+
<div class="stat"><span>Training now</span><b id="s-run">—</b><small>on workers</small></div>
|
| 19 |
+
<div class="stat"><span>Trainers</span><b id="s-users">—</b><small>people</small></div>
|
| 20 |
+
<div class="stat"><span>Tokens burned</span><b id="s-tok">—</b><small>all time</small></div>
|
| 21 |
+
</div>
|
| 22 |
+
|
| 23 |
+
<section id="liveSec" style="margin-bottom:32px;display:none">
|
| 24 |
+
<div class="sect-h"><h2>Being born right now</h2></div>
|
| 25 |
+
<div class="runs" id="live"></div>
|
| 26 |
+
</section>
|
| 27 |
+
|
| 28 |
+
<section style="margin-bottom:32px">
|
| 29 |
+
<div class="sect-h" style="margin-bottom:12px">
|
| 30 |
+
<h2>Best loss by size</h2>
|
| 31 |
+
<span class="sub">lower is better</span>
|
| 32 |
+
</div>
|
| 33 |
+
<div id="board" class="grid g2"></div>
|
| 34 |
+
</section>
|
| 35 |
+
|
| 36 |
+
<section style="margin-bottom:32px">
|
| 37 |
+
<div class="sect-h" style="margin-bottom:12px"><h2>All models</h2></div>
|
| 38 |
+
<div style="display:flex;gap:7px;flex-wrap:wrap;margin-bottom:8px" id="tierFilter">
|
| 39 |
+
<button class="btn sm primary" data-tier="">All sizes</button>
|
| 40 |
+
{% for t in tiers %}
|
| 41 |
+
<button class="btn sm" data-tier="{{ t.key }}">{{ t.label }}</button>
|
| 42 |
+
{% endfor %}
|
| 43 |
+
</div>
|
| 44 |
+
<div style="display:flex;gap:7px;flex-wrap:wrap;margin-bottom:18px" id="sortFilter">
|
| 45 |
+
<button class="btn sm primary" data-sort="recent">Newest</button>
|
| 46 |
+
<button class="btn sm" data-sort="loss">Lowest loss</button>
|
| 47 |
+
<button class="btn sm" data-sort="tokens">Most tokens</button>
|
| 48 |
+
<button class="btn sm" data-sort="params">Largest</button>
|
| 49 |
+
</div>
|
| 50 |
+
<div id="models"></div>
|
| 51 |
+
</section>
|
| 52 |
+
|
| 53 |
+
<section>
|
| 54 |
+
<div class="sect-h" style="margin-bottom:12px"><h2>Trainers</h2></div>
|
| 55 |
+
<div id="trainers" class="grid g3"></div>
|
| 56 |
+
</section>
|
| 57 |
+
|
| 58 |
+
{% endblock %}
|
| 59 |
+
{% block scripts %}
|
| 60 |
+
<script>
|
| 61 |
+
let tier = '', sort = 'recent';
|
| 62 |
+
|
| 63 |
+
function card(m){
|
| 64 |
+
return `<a class="run" href="/models/${m.id}">
|
| 65 |
+
<div class="run-top">
|
| 66 |
+
<h4>${NX.esc(m.name)}</h4>
|
| 67 |
+
${NX.chip(m.status)}
|
| 68 |
+
</div>
|
| 69 |
+
<div class="run-meta">
|
| 70 |
+
<span class="mono">${NX.esc(m.tier_label)}</span>
|
| 71 |
+
<span>${Number(m.n_params).toLocaleString()} params</span>
|
| 72 |
+
<span><b>${NX.fmtTokens(m.tokens_seen)}</b> tokens</span>
|
| 73 |
+
${m.best_loss ? `<span>loss <b>${m.best_loss.toFixed(4)}</b></span>` : ''}
|
| 74 |
+
<span>@${NX.esc(m.username)}</span>
|
| 75 |
+
<span class="muted">${NX.ago(m.finished_at || m.created_at)}</span>
|
| 76 |
+
</div>
|
| 77 |
+
</a>`;
|
| 78 |
+
}
|
| 79 |
+
|
| 80 |
+
function bind(sel, key, fn){
|
| 81 |
+
document.querySelectorAll(sel + ' .btn').forEach(b =>
|
| 82 |
+
b.addEventListener('click', () => {
|
| 83 |
+
document.querySelectorAll(sel + ' .btn').forEach(x => x.classList.remove('primary'));
|
| 84 |
+
b.classList.add('primary');
|
| 85 |
+
fn(b.dataset[key]);
|
| 86 |
+
loadModels();
|
| 87 |
+
}));
|
| 88 |
+
}
|
| 89 |
+
bind('#tierFilter', 'tier', v => tier = v);
|
| 90 |
+
bind('#sortFilter', 'sort', v => sort = v);
|
| 91 |
+
|
| 92 |
+
async function loadModels(){
|
| 93 |
+
const q = await NX.get(`/api/explore?tier=${tier}&sort=${sort}&limit=60`);
|
| 94 |
+
|
| 95 |
+
document.getElementById('s-done').textContent = q.stats.done;
|
| 96 |
+
document.getElementById('s-run').textContent = q.stats.running;
|
| 97 |
+
document.getElementById('s-users').textContent = q.stats.users;
|
| 98 |
+
document.getElementById('s-tok').textContent = NX.fmtTokens(q.stats.tokens);
|
| 99 |
+
|
| 100 |
+
const liveSec = document.getElementById('liveSec');
|
| 101 |
+
liveSec.style.display = q.active.length ? 'block' : 'none';
|
| 102 |
+
document.getElementById('live').innerHTML = q.active.map(NX.runRow).join('');
|
| 103 |
+
|
| 104 |
+
document.getElementById('models').innerHTML = q.models.length
|
| 105 |
+
? `<div class="runs">${q.models.map(card).join('')}</div>`
|
| 106 |
+
: `<div class="empty">No finished models${tier ? ' at this size' : ''} yet.</div>`;
|
| 107 |
+
}
|
| 108 |
+
|
| 109 |
+
async function loadBoard(){
|
| 110 |
+
const {tiers} = await NX.get('/api/leaderboard');
|
| 111 |
+
const el = document.getElementById('board');
|
| 112 |
+
if(!tiers.length){
|
| 113 |
+
el.innerHTML = `<div class="empty" style="grid-column:1/-1">
|
| 114 |
+
Nothing finished yet — the first model trained here takes every record.</div>`;
|
| 115 |
+
return;
|
| 116 |
+
}
|
| 117 |
+
el.innerHTML = tiers.map(t => `<div class="card">
|
| 118 |
+
<div class="eyebrow" style="margin-bottom:10px">${NX.esc(t.label)}</div>
|
| 119 |
+
${t.models.map((m,i) => `<div class="kv">
|
| 120 |
+
<span><b class="mono" style="color:var(--accent)">#${i+1}</b>
|
| 121 |
+
<a href="/models/${m.id}" style="color:var(--fg)">${NX.esc(m.name)}</a>
|
| 122 |
+
<span class="muted">@${NX.esc(m.username)}</span></span>
|
| 123 |
+
<b>${m.best_loss.toFixed(4)}</b></div>`).join('')}
|
| 124 |
+
</div>`).join('');
|
| 125 |
+
}
|
| 126 |
+
|
| 127 |
+
async function loadTrainers(){
|
| 128 |
+
const {trainers} = await NX.get('/api/trainers');
|
| 129 |
+
document.getElementById('trainers').innerHTML = trainers.length
|
| 130 |
+
? trainers.map(t => `<a class="card" href="/u/${encodeURIComponent(t.username)}"
|
| 131 |
+
style="display:flex;gap:12px;align-items:center">
|
| 132 |
+
${t.avatar_url
|
| 133 |
+
? `<img class="avatar" style="width:40px;height:40px" src="${NX.esc(t.avatar_url)}" alt="">`
|
| 134 |
+
: `<span class="avatar" style="width:40px;height:40px"></span>`}
|
| 135 |
+
<div style="min-width:0">
|
| 136 |
+
<div style="font-weight:620">${NX.esc(t.display_name || t.username)}</div>
|
| 137 |
+
<div class="sub" style="font-size:12.5px">
|
| 138 |
+
${t.done} model${t.done === 1 ? '' : 's'} ·
|
| 139 |
+
${NX.fmtTokens(t.tokens)} tokens${t.best_loss
|
| 140 |
+
? ' · best ' + t.best_loss.toFixed(3) : ''}</div>
|
| 141 |
+
</div></a>`).join('')
|
| 142 |
+
: `<div class="empty" style="grid-column:1/-1">No trainers yet.</div>`;
|
| 143 |
+
}
|
| 144 |
+
|
| 145 |
+
NX.poll(async () => { await loadModels(); await loadBoard(); await loadTrainers(); }, 8000);
|
| 146 |
+
</script>
|
| 147 |
+
{% endblock %}
|
web/templates/home.html
ADDED
|
@@ -0,0 +1,71 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{% extends "base.html" %}
|
| 2 |
+
{% set nav = 'home' %}
|
| 3 |
+
{% block title %}NanoDex{% endblock %}
|
| 4 |
+
{% block body %}
|
| 5 |
+
|
| 6 |
+
<div class="sect-h" style="margin-bottom:20px">
|
| 7 |
+
<div>
|
| 8 |
+
<span class="eyebrow">Welcome back</span>
|
| 9 |
+
<h1 style="font-size:28px;margin-top:5px">{{ user.name or user.username }}</h1>
|
| 10 |
+
</div>
|
| 11 |
+
<a class="btn primary" href="/train">+ New run</a>
|
| 12 |
+
</div>
|
| 13 |
+
|
| 14 |
+
<div class="grid g4" id="tiles">
|
| 15 |
+
<div class="stat"><span>Your runs</span><b id="t-mine">—</b><small id="t-mine-s"> </small></div>
|
| 16 |
+
<div class="stat"><span>In flight</span><b id="t-active">—</b><small id="t-active-s"> </small></div>
|
| 17 |
+
<div class="stat"><span>Tokens trained</span><b id="t-tok">—</b><small>across all your runs</small></div>
|
| 18 |
+
<div class="stat"><span>Best loss</span><b id="t-loss">—</b><small id="t-loss-s"> </small></div>
|
| 19 |
+
</div>
|
| 20 |
+
|
| 21 |
+
<section style="margin-top:34px">
|
| 22 |
+
<div class="sect-h">
|
| 23 |
+
<h2>Live now</h2>
|
| 24 |
+
<a class="btn ghost sm" href="/queue">Global queue →</a>
|
| 25 |
+
</div>
|
| 26 |
+
<div id="live" class="runs"></div>
|
| 27 |
+
</section>
|
| 28 |
+
|
| 29 |
+
<section style="margin-top:34px">
|
| 30 |
+
<div class="sect-h">
|
| 31 |
+
<h2>Recent runs</h2>
|
| 32 |
+
<a class="btn ghost sm" href="/models">All models →</a>
|
| 33 |
+
</div>
|
| 34 |
+
<div id="recent" class="runs"></div>
|
| 35 |
+
</section>
|
| 36 |
+
|
| 37 |
+
{% endblock %}
|
| 38 |
+
{% block scripts %}
|
| 39 |
+
<script>
|
| 40 |
+
async function refresh(){
|
| 41 |
+
const {jobs} = await NX.get('/api/jobs');
|
| 42 |
+
const active = jobs.filter(j => j.status==='running' || j.status==='queued');
|
| 43 |
+
const done = jobs.filter(j => j.status==='done');
|
| 44 |
+
|
| 45 |
+
document.getElementById('t-mine').textContent = jobs.length;
|
| 46 |
+
document.getElementById('t-mine-s').textContent = done.length + ' finished';
|
| 47 |
+
document.getElementById('t-active').textContent = active.length;
|
| 48 |
+
document.getElementById('t-active-s').textContent =
|
| 49 |
+
active.filter(j=>j.status==='running').length + ' training now';
|
| 50 |
+
document.getElementById('t-tok').textContent =
|
| 51 |
+
NX.fmtTokens(jobs.reduce((a,j)=>a+(j.tokens_seen||0),0));
|
| 52 |
+
const best = done.map(j=>j.best_loss).filter(Boolean);
|
| 53 |
+
document.getElementById('t-loss').textContent =
|
| 54 |
+
best.length ? Math.min(...best).toFixed(4) : '—';
|
| 55 |
+
document.getElementById('t-loss-s').textContent =
|
| 56 |
+
best.length ? 'cross-entropy' : 'no finished runs yet';
|
| 57 |
+
|
| 58 |
+
const live = document.getElementById('live');
|
| 59 |
+
live.innerHTML = active.length
|
| 60 |
+
? active.map(NX.runRow).join('')
|
| 61 |
+
: `<div class="empty">Nothing of yours is training right now.
|
| 62 |
+
<a href="/train" style="color:var(--accent)">Start a run →</a></div>`;
|
| 63 |
+
|
| 64 |
+
const rec = jobs.filter(j => j.status!=='running' && j.status!=='queued').slice(0,6);
|
| 65 |
+
document.getElementById('recent').innerHTML = rec.length
|
| 66 |
+
? rec.map(NX.runRow).join('')
|
| 67 |
+
: `<div class="empty">No finished runs yet — your first model is one click away.</div>`;
|
| 68 |
+
}
|
| 69 |
+
NX.poll(refresh, 4000);
|
| 70 |
+
</script>
|
| 71 |
+
{% endblock %}
|
web/templates/landing.html
ADDED
|
@@ -0,0 +1,128 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{% extends "base.html" %}
|
| 2 |
+
{% block title %}NanoDex — train a language model from scratch{% endblock %}
|
| 3 |
+
{% block body %}
|
| 4 |
+
|
| 5 |
+
<section class="hero">
|
| 6 |
+
<span class="tag"><b>{{ n_devices }}</b> {{ 'GPU' if on_gpu else 'CPU' }} worker{{ 's' if n_devices != 1 }} ·
|
| 7 |
+
<b>{{ stats.done }}</b> models born here</span>
|
| 8 |
+
<h1>Train your first<br><em>decoder-only model</em><br>from scratch.</h1>
|
| 9 |
+
<p>Not a fine-tune. Not an adapter. Random weights, a real corpus, and a real
|
| 10 |
+
loss curve you watch fall in real time — from a browser tab, on our GPUs.</p>
|
| 11 |
+
<div class="cta">
|
| 12 |
+
<a class="btn primary lg" target="_top" href="/login?next=/train">Start a run →</a>
|
| 13 |
+
<a class="btn lg" href="/explore">Browse the models</a>
|
| 14 |
+
</div>
|
| 15 |
+
<p class="frame-note">Signing in opens NanoDex full screen — Hugging Face's
|
| 16 |
+
login page can't run inside an embedded frame.</p>
|
| 17 |
+
</section>
|
| 18 |
+
|
| 19 |
+
<section style="margin-top:22px">
|
| 20 |
+
<div class="grid g3">
|
| 21 |
+
<div class="card">
|
| 22 |
+
<div class="step-n">1</div>
|
| 23 |
+
<h3>Pick a size</h3>
|
| 24 |
+
<p class="sub" style="margin:6px 0 0">Four architectures, from 492k to 8.1M
|
| 25 |
+
parameters. That's the <em>real</em> total — embeddings included, no asterisk.</p>
|
| 26 |
+
</div>
|
| 27 |
+
<div class="card">
|
| 28 |
+
<div class="step-n">2</div>
|
| 29 |
+
<h3>Pick a token budget</h3>
|
| 30 |
+
<p class="sub" style="margin:6px 0 0">Anywhere from
|
| 31 |
+
{{ min_tokens // 10**6 }}M to <b>1.5 billion</b> tokens of
|
| 32 |
+
<b>fineweb-edu</b>. More tokens, sharper model, longer wait.</p>
|
| 33 |
+
</div>
|
| 34 |
+
<div class="card">
|
| 35 |
+
<div class="step-n">3</div>
|
| 36 |
+
<h3>Watch it learn</h3>
|
| 37 |
+
<p class="sub" style="margin:6px 0 0">Your run joins the queue. Live loss
|
| 38 |
+
curve, throughput, logs — then the weights land in your own Hugging Face
|
| 39 |
+
account.</p>
|
| 40 |
+
</div>
|
| 41 |
+
</div>
|
| 42 |
+
</section>
|
| 43 |
+
|
| 44 |
+
<section style="margin-top:44px">
|
| 45 |
+
<div class="sect-h">
|
| 46 |
+
<div>
|
| 47 |
+
<span class="eyebrow">The four sizes</span>
|
| 48 |
+
<h2 style="margin-top:5px">Small enough to finish. Big enough to learn something.</h2>
|
| 49 |
+
</div>
|
| 50 |
+
</div>
|
| 51 |
+
<div class="picks">
|
| 52 |
+
{% for t in tiers %}
|
| 53 |
+
<div class="pick">
|
| 54 |
+
<b>{{ t.label }}</b>
|
| 55 |
+
<div class="p-n">{{ "{:,}".format(t.params) }} params</div>
|
| 56 |
+
<div class="p-d">
|
| 57 |
+
{{ t.layers }} layers · {{ t.hidden }} hidden<br>
|
| 58 |
+
{{ t.heads }} heads ({{ t.kv_heads }} KV) · FFN {{ t.ffn }}
|
| 59 |
+
</div>
|
| 60 |
+
</div>
|
| 61 |
+
{% endfor %}
|
| 62 |
+
</div>
|
| 63 |
+
</section>
|
| 64 |
+
|
| 65 |
+
<section style="margin-top:44px">
|
| 66 |
+
<div class="grid g2">
|
| 67 |
+
<div class="card pad-lg">
|
| 68 |
+
<span class="eyebrow">What you actually get</span>
|
| 69 |
+
<h2 style="margin:8px 0 12px;font-size:21px">A model that is genuinely yours</h2>
|
| 70 |
+
<p class="sub">Every finished run is pushed to <b>your</b> Hugging Face
|
| 71 |
+
namespace: <span class="mono">config.json</span>,
|
| 72 |
+
<span class="mono">model.safetensors</span>, the tokenizer, a
|
| 73 |
+
<span class="mono">training_run.json</span> with the full recipe, and a
|
| 74 |
+
generated model card. Load it with <span class="mono">transformers</span>
|
| 75 |
+
like anything else on the Hub.</p>
|
| 76 |
+
</div>
|
| 77 |
+
<div class="card pad-lg">
|
| 78 |
+
<span class="eyebrow">Honest expectations</span>
|
| 79 |
+
<h2 style="margin:8px 0 12px;font-size:21px">It will not be an assistant</h2>
|
| 80 |
+
<p class="sub">At a few hundred thousand parameters, a model learns word
|
| 81 |
+
shapes, common collocations and a little syntax. It will produce
|
| 82 |
+
English-looking text with no facts in it. That's the correct outcome —
|
| 83 |
+
and watching cross-entropy fall from <b>7.6</b> (uniform noise over 2,048
|
| 84 |
+
tokens) to somewhere near <b>4</b> is the whole point.</p>
|
| 85 |
+
</div>
|
| 86 |
+
</div>
|
| 87 |
+
</section>
|
| 88 |
+
|
| 89 |
+
{% if recent %}
|
| 90 |
+
<section style="margin-top:44px">
|
| 91 |
+
<div class="sect-h">
|
| 92 |
+
<div>
|
| 93 |
+
<span class="eyebrow">Trained here</span>
|
| 94 |
+
<h2 style="margin-top:5px">The newest models on the shelf</h2>
|
| 95 |
+
</div>
|
| 96 |
+
<a class="btn ghost sm" href="/explore">Explore all →</a>
|
| 97 |
+
</div>
|
| 98 |
+
<div class="runs">
|
| 99 |
+
{% for m in recent %}
|
| 100 |
+
<a class="run" href="/models/{{ m.id }}">
|
| 101 |
+
<div class="run-top">
|
| 102 |
+
<h4>{{ m.name }}</h4>
|
| 103 |
+
<span class="chip done"><i></i>completed</span>
|
| 104 |
+
</div>
|
| 105 |
+
<div class="run-meta">
|
| 106 |
+
<span class="mono">{{ m.tier_label }}</span>
|
| 107 |
+
<span>{{ "{:,}".format(m.n_params) }} params</span>
|
| 108 |
+
<span><b>{{ (m.tokens_seen / 1000000)|round|int }}M</b> tokens</span>
|
| 109 |
+
{% if m.best_loss %}<span>loss <b>{{ "%.4f"|format(m.best_loss) }}</b></span>{% endif %}
|
| 110 |
+
<span>@{{ m.username }}</span>
|
| 111 |
+
</div>
|
| 112 |
+
</a>
|
| 113 |
+
{% endfor %}
|
| 114 |
+
</div>
|
| 115 |
+
</section>
|
| 116 |
+
{% endif %}
|
| 117 |
+
|
| 118 |
+
<section style="margin-top:44px" class="card pad-lg">
|
| 119 |
+
<div style="display:flex;gap:24px;align-items:center;justify-content:space-between;flex-wrap:wrap">
|
| 120 |
+
<div>
|
| 121 |
+
<h2 style="font-size:21px">Ready when you are.</h2>
|
| 122 |
+
<p class="sub" style="margin-top:6px">Sign in with Hugging Face — no separate account, no password.</p>
|
| 123 |
+
</div>
|
| 124 |
+
<a class="btn primary lg" target="_top" href="/login?next=/train">Start a run →</a>
|
| 125 |
+
</div>
|
| 126 |
+
</section>
|
| 127 |
+
|
| 128 |
+
{% endblock %}
|
web/templates/models.html
ADDED
|
@@ -0,0 +1,75 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{% extends "base.html" %}
|
| 2 |
+
{% set nav = 'models' %}
|
| 3 |
+
{% block title %}Models — NanoDex{% endblock %}
|
| 4 |
+
{% block body %}
|
| 5 |
+
|
| 6 |
+
<div class="sect-h" style="margin-bottom:22px">
|
| 7 |
+
<div>
|
| 8 |
+
<span class="eyebrow">Your models</span>
|
| 9 |
+
<h1 style="font-size:28px;margin-top:5px">Everything you've trained</h1>
|
| 10 |
+
</div>
|
| 11 |
+
<a class="btn primary" href="/train">+ New run</a>
|
| 12 |
+
</div>
|
| 13 |
+
|
| 14 |
+
<div style="display:flex;gap:7px;flex-wrap:wrap;margin-bottom:22px" id="filters">
|
| 15 |
+
<button class="btn sm on" data-f="all">All</button>
|
| 16 |
+
<button class="btn sm" data-f="running">Training</button>
|
| 17 |
+
<button class="btn sm" data-f="queued">Queued</button>
|
| 18 |
+
<button class="btn sm" data-f="done">Completed</button>
|
| 19 |
+
<button class="btn sm" data-f="other">Failed / cancelled</button>
|
| 20 |
+
</div>
|
| 21 |
+
|
| 22 |
+
<section id="sec-running" style="margin-bottom:32px"></section>
|
| 23 |
+
<section id="sec-queued" style="margin-bottom:32px"></section>
|
| 24 |
+
<section id="sec-done" style="margin-bottom:32px"></section>
|
| 25 |
+
<section id="sec-other"></section>
|
| 26 |
+
<div id="none"></div>
|
| 27 |
+
|
| 28 |
+
{% endblock %}
|
| 29 |
+
{% block scripts %}
|
| 30 |
+
<script>
|
| 31 |
+
let filter = 'all';
|
| 32 |
+
|
| 33 |
+
document.querySelectorAll('#filters .btn').forEach(b =>
|
| 34 |
+
b.addEventListener('click', () => {
|
| 35 |
+
document.querySelectorAll('#filters .btn').forEach(x =>
|
| 36 |
+
x.classList.remove('primary'));
|
| 37 |
+
b.classList.add('primary');
|
| 38 |
+
filter = b.dataset.f;
|
| 39 |
+
render(last);
|
| 40 |
+
}));
|
| 41 |
+
document.querySelector('#filters .btn').classList.add('primary');
|
| 42 |
+
|
| 43 |
+
let last = [];
|
| 44 |
+
|
| 45 |
+
function section(id, title, note, jobs){
|
| 46 |
+
const el = document.getElementById(id);
|
| 47 |
+
if(!jobs.length){ el.innerHTML = ''; return 0; }
|
| 48 |
+
el.innerHTML = `<div class="sect-h">
|
| 49 |
+
<div><h2>${title} <span class="muted" style="font-weight:400">
|
| 50 |
+
(${jobs.length})</span></h2>
|
| 51 |
+
${note ? `<p class="sub" style="margin-top:3px">${note}</p>` : ''}</div>
|
| 52 |
+
</div>
|
| 53 |
+
<div class="runs">${jobs.map(NX.runRow).join('')}</div>`;
|
| 54 |
+
return jobs.length;
|
| 55 |
+
}
|
| 56 |
+
|
| 57 |
+
function render(jobs){
|
| 58 |
+
last = jobs;
|
| 59 |
+
const pick = s => (filter==='all' || filter===s) ? jobs.filter(j =>
|
| 60 |
+
s==='other' ? (j.status==='failed'||j.status==='cancelled') : j.status===s) : [];
|
| 61 |
+
|
| 62 |
+
let n = 0;
|
| 63 |
+
n += section('sec-running','Training now','Live on a worker right now.', pick('running'));
|
| 64 |
+
n += section('sec-queued','In the queue','Waiting for a free worker — oldest first.', pick('queued'));
|
| 65 |
+
n += section('sec-done','Completed','Finished. Open one to publish it or generate text.', pick('done'));
|
| 66 |
+
n += section('sec-other','Failed or cancelled','', pick('other'));
|
| 67 |
+
|
| 68 |
+
document.getElementById('none').innerHTML = n ? '' :
|
| 69 |
+
`<div class="empty">Nothing here yet.
|
| 70 |
+
<a href="/train" style="color:var(--accent)">Train your first model →</a></div>`;
|
| 71 |
+
}
|
| 72 |
+
|
| 73 |
+
NX.poll(async () => render((await NX.get('/api/jobs')).jobs), 4000);
|
| 74 |
+
</script>
|
| 75 |
+
{% endblock %}
|
web/templates/profile.html
ADDED
|
@@ -0,0 +1,74 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{% extends "base.html" %}
|
| 2 |
+
{% block title %}{{ user.username }} — NanoDex{% endblock %}
|
| 3 |
+
{% block body %}
|
| 4 |
+
|
| 5 |
+
<div class="card pad-lg" style="display:flex;gap:20px;align-items:center;flex-wrap:wrap">
|
| 6 |
+
{% if user.picture %}
|
| 7 |
+
<img src="{{ user.picture }}" alt="" style="width:72px;height:72px;border-radius:50%;
|
| 8 |
+
border:1px solid var(--line-2);object-fit:cover">
|
| 9 |
+
{% else %}
|
| 10 |
+
<div style="width:72px;height:72px;border-radius:50%;background:var(--panel-2);
|
| 11 |
+
border:1px solid var(--line-2)"></div>
|
| 12 |
+
{% endif %}
|
| 13 |
+
<div style="flex:1;min-width:200px">
|
| 14 |
+
<h1 style="font-size:24px">{{ user.name or user.username }}</h1>
|
| 15 |
+
<p class="sub" style="margin-top:3px">
|
| 16 |
+
<a href="https://huggingface.co/{{ user.username }}" target="_blank" rel="noopener"
|
| 17 |
+
class="mono" style="color:var(--fg-2)">@{{ user.username }}</a>
|
| 18 |
+
· signed in with Hugging Face
|
| 19 |
+
</p>
|
| 20 |
+
</div>
|
| 21 |
+
<div style="display:flex;gap:8px;flex-wrap:wrap">
|
| 22 |
+
<a class="btn" href="/u/{{ user.username }}">Public profile</a>
|
| 23 |
+
<a class="btn" href="/models">My models</a>
|
| 24 |
+
<a class="btn ghost" href="/logout">Sign out</a>
|
| 25 |
+
</div>
|
| 26 |
+
</div>
|
| 27 |
+
|
| 28 |
+
<div class="grid g4" style="margin-top:16px">
|
| 29 |
+
<div class="stat"><span>Runs</span><b>{{ summary.total }}</b>
|
| 30 |
+
<small>{{ summary.done }} finished · {{ summary.active }} in flight</small></div>
|
| 31 |
+
<div class="stat"><span>Tokens trained</span>
|
| 32 |
+
<b>{{ "%.0fM"|format(summary.tokens / 1000000) if summary.tokens < 1000000000 else "%.2fB"|format(summary.tokens / 1000000000) }}</b>
|
| 33 |
+
<small>of fineweb-edu</small></div>
|
| 34 |
+
<div class="stat"><span>Parameters created</span>
|
| 35 |
+
<b>{{ "{:,}".format(summary.params) }}</b><small>across finished models</small></div>
|
| 36 |
+
<div class="stat"><span>Best loss</span>
|
| 37 |
+
<b>{{ "%.4f"|format(summary.best) if summary.best else "—" }}</b>
|
| 38 |
+
<small>{{ summary.published }} published to the Hub</small></div>
|
| 39 |
+
</div>
|
| 40 |
+
|
| 41 |
+
<section style="margin-top:34px">
|
| 42 |
+
<div class="sect-h">
|
| 43 |
+
<h2>Recent activity</h2>
|
| 44 |
+
<a class="btn ghost sm" href="/models">All models →</a>
|
| 45 |
+
</div>
|
| 46 |
+
<div id="runs" class="runs"></div>
|
| 47 |
+
</section>
|
| 48 |
+
|
| 49 |
+
<section style="margin-top:34px">
|
| 50 |
+
<div class="card">
|
| 51 |
+
<h2 style="font-size:16px;margin-bottom:6px">Permissions</h2>
|
| 52 |
+
<p class="sub">NanoDex uses your Hugging Face sign-in for two things: to know
|
| 53 |
+
which runs are yours, and — only when you press Publish — to upload a
|
| 54 |
+
finished model to your namespace. Nothing is written to your account
|
| 55 |
+
otherwise, and the session expires after 8 hours.</p>
|
| 56 |
+
<p class="sub" style="margin-top:12px">Your finished models are visible to
|
| 57 |
+
everyone who can open NanoDex, on your
|
| 58 |
+
<a href="/u/{{ user.username }}" style="color:var(--accent)">public
|
| 59 |
+
profile</a> and in <a href="/explore" style="color:var(--accent)">Explore</a>,
|
| 60 |
+
and anyone signed in can generate text from them. Cancelling a run and
|
| 61 |
+
publishing to the Hub stay yours alone.</p>
|
| 62 |
+
</div>
|
| 63 |
+
</section>
|
| 64 |
+
|
| 65 |
+
{% endblock %}
|
| 66 |
+
{% block scripts %}
|
| 67 |
+
<script>
|
| 68 |
+
const JOBS = {{ jobs | tojson }};
|
| 69 |
+
document.getElementById('runs').innerHTML = JOBS.length
|
| 70 |
+
? JOBS.slice(0,10).map(NX.runRow).join('')
|
| 71 |
+
: `<div class="empty">No runs yet.
|
| 72 |
+
<a href="/train" style="color:var(--accent)">Train your first model →</a></div>`;
|
| 73 |
+
</script>
|
| 74 |
+
{% endblock %}
|
web/templates/queue.html
ADDED
|
@@ -0,0 +1,120 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{% extends "base.html" %}
|
| 2 |
+
{% set nav = 'queue' %}
|
| 3 |
+
{% block title %}Queue — NanoDex{% endblock %}
|
| 4 |
+
{% block body %}
|
| 5 |
+
|
| 6 |
+
<div class="sect-h" style="margin-bottom:22px">
|
| 7 |
+
<div>
|
| 8 |
+
<span class="eyebrow">Shared infrastructure</span>
|
| 9 |
+
<h1 style="font-size:28px;margin-top:5px">The queue, right now</h1>
|
| 10 |
+
</div>
|
| 11 |
+
{% if user %}<a class="btn primary" href="/train">+ New run</a>
|
| 12 |
+
{% else %}<a class="btn primary" target="_top" href="/login?next=/train">Start a run</a>{% endif %}
|
| 13 |
+
</div>
|
| 14 |
+
|
| 15 |
+
<div class="grid g4" style="margin-bottom:20px">
|
| 16 |
+
<div class="stat"><span>Training</span><b id="s-run">—</b><small>on workers</small></div>
|
| 17 |
+
<div class="stat"><span>Waiting</span><b id="s-q">—</b><small>in the queue</small></div>
|
| 18 |
+
<div class="stat"><span>Models born</span><b id="s-done">—</b><small>finished runs</small></div>
|
| 19 |
+
<div class="stat"><span>Tokens burned</span><b id="s-tok">—</b><small>all time</small></div>
|
| 20 |
+
</div>
|
| 21 |
+
|
| 22 |
+
<section style="margin-bottom:30px">
|
| 23 |
+
<div class="sect-h"><h2>Workers</h2></div>
|
| 24 |
+
<div class="grid g3" id="workers"></div>
|
| 25 |
+
<p class="hint" id="cache"></p>
|
| 26 |
+
<p class="hint" id="storage"></p>
|
| 27 |
+
</section>
|
| 28 |
+
|
| 29 |
+
<section style="margin-bottom:30px">
|
| 30 |
+
<div class="sect-h"><h2>Waiting in line</h2></div>
|
| 31 |
+
<div id="queued"></div>
|
| 32 |
+
</section>
|
| 33 |
+
|
| 34 |
+
<section>
|
| 35 |
+
<div class="sect-h"><h2>Recently finished</h2></div>
|
| 36 |
+
<div id="recent"></div>
|
| 37 |
+
</section>
|
| 38 |
+
|
| 39 |
+
{% endblock %}
|
| 40 |
+
{% block scripts %}
|
| 41 |
+
<script>
|
| 42 |
+
function tbl(rows, cols){
|
| 43 |
+
if(!rows.length) return '';
|
| 44 |
+
return `<div class="card tscroll" style="padding:6px 4px 4px"><table>
|
| 45 |
+
<thead><tr>${cols.map(c=>`<th>${c[0]}</th>`).join('')}</tr></thead>
|
| 46 |
+
<tbody>${rows.map(r=>`<tr>${cols.map(c=>`<td>${c[1](r)}</td>`).join('')}</tr>`).join('')}</tbody>
|
| 47 |
+
</table></div>`;
|
| 48 |
+
}
|
| 49 |
+
|
| 50 |
+
async function refresh(){
|
| 51 |
+
const q = await NX.get('/api/queue');
|
| 52 |
+
|
| 53 |
+
document.getElementById('s-run').textContent = q.stats.running;
|
| 54 |
+
document.getElementById('s-q').textContent = q.stats.queued;
|
| 55 |
+
document.getElementById('s-done').textContent = q.stats.done;
|
| 56 |
+
document.getElementById('s-tok').textContent = NX.fmtTokens(q.stats.tokens);
|
| 57 |
+
|
| 58 |
+
document.getElementById('workers').innerHTML = q.workers.map(w => {
|
| 59 |
+
if(!w.job){
|
| 60 |
+
return `<div class="gpu">
|
| 61 |
+
<div class="gpu-h"><b>Worker ${w.index}</b><span>${NX.esc(w.device)}</span></div>
|
| 62 |
+
<div class="muted" style="font-size:13.5px">idle — ready for the next run</div>
|
| 63 |
+
</div>`;
|
| 64 |
+
}
|
| 65 |
+
const j = w.job, pct = (j.progress*100).toFixed(1);
|
| 66 |
+
return `<div class="gpu busy">
|
| 67 |
+
<div class="gpu-h"><b>Worker ${w.index}</b><span>${NX.esc(w.device)}</span></div>
|
| 68 |
+
<div style="font-weight:620;font-size:14.5px">${NX.esc(j.name)}</div>
|
| 69 |
+
<div class="sub" style="font-size:12.5px">@${NX.esc(j.username)} · ${NX.esc(j.tier_label)}</div>
|
| 70 |
+
<div class="bar thin" style="margin:10px 0 7px"><i style="width:${pct}%"></i></div>
|
| 71 |
+
<div class="mono muted" style="font-size:12px">${pct}% ·
|
| 72 |
+
loss ${j.loss ? j.loss.toFixed(3) : '—'} ·
|
| 73 |
+
${Math.round(j.tok_per_s/1e3)}k tok/s</div>
|
| 74 |
+
</div>`;
|
| 75 |
+
}).join('');
|
| 76 |
+
|
| 77 |
+
const d = q.data;
|
| 78 |
+
const pct = Math.min(100, 100 * d.ready_tokens / (d.target||1));
|
| 79 |
+
document.getElementById('cache').textContent =
|
| 80 |
+
`fineweb-edu token cache: ${NX.fmtTokens(d.ready_tokens)} tokens ready ` +
|
| 81 |
+
`(${pct.toFixed(0)}% · ${d.status})`;
|
| 82 |
+
|
| 83 |
+
const st = q.store || {};
|
| 84 |
+
document.getElementById('storage').innerHTML = st.enabled
|
| 85 |
+
? `durable storage: <b>on</b> — finished models and their loss curves are
|
| 86 |
+
archived to <span class="mono">${NX.esc(st.bucket)}</span> and survive
|
| 87 |
+
restarts${st.restored ? ` (${st.restored} run${st.restored===1?'':'s'} restored on boot)` : ''}`
|
| 88 |
+
: `durable storage: <b style="color:var(--amber)">off</b> — this container's
|
| 89 |
+
history and checkpoints are lost on the next restart`;
|
| 90 |
+
|
| 91 |
+
document.getElementById('queued').innerHTML = q.queued.length
|
| 92 |
+
? tbl(q.queued, [
|
| 93 |
+
['#', r => '<b>'+(r.queue_position||'?')+'</b>'],
|
| 94 |
+
['Model', r => NX.esc(r.name)],
|
| 95 |
+
['Author', r => `<a href="/u/${encodeURIComponent(r.username)}"
|
| 96 |
+
style="color:var(--accent)">@${NX.esc(r.username)}</a>`],
|
| 97 |
+
['Size', r => NX.esc(r.tier_label)],
|
| 98 |
+
['Tokens', r => NX.fmtTokens(r.target_tokens)],
|
| 99 |
+
['Waiting', r => NX.ago(r.created_at)],
|
| 100 |
+
])
|
| 101 |
+
: `<div class="empty">Queue is empty — a worker will start your run immediately.</div>`;
|
| 102 |
+
|
| 103 |
+
document.getElementById('recent').innerHTML = q.recent.length
|
| 104 |
+
? tbl(q.recent, [
|
| 105 |
+
['Model', r => NX.esc(r.name)],
|
| 106 |
+
['Author', r => `<a href="/u/${encodeURIComponent(r.username)}"
|
| 107 |
+
style="color:var(--accent)">@${NX.esc(r.username)}</a>`],
|
| 108 |
+
['Size', r => NX.esc(r.tier_label)],
|
| 109 |
+
['Tokens', r => NX.fmtTokens(r.tokens_seen)],
|
| 110 |
+
['Final loss', r => r.loss ? r.loss.toFixed(4) : '—'],
|
| 111 |
+
['On the Hub', r => r.pushed_repo
|
| 112 |
+
? `<a class="mono" style="color:var(--accent)" target="_blank" rel="noopener"
|
| 113 |
+
href="https://huggingface.co/${NX.esc(r.pushed_repo)}">${NX.esc(r.pushed_repo)}</a>`
|
| 114 |
+
: '<span class="muted">not published</span>'],
|
| 115 |
+
])
|
| 116 |
+
: `<div class="empty">No finished runs yet. Someone has to be first.</div>`;
|
| 117 |
+
}
|
| 118 |
+
NX.poll(refresh, 4000);
|
| 119 |
+
</script>
|
| 120 |
+
{% endblock %}
|
web/templates/train.html
ADDED
|
@@ -0,0 +1,273 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{% extends "base.html" %}
|
| 2 |
+
{% set nav = 'train' %}
|
| 3 |
+
{% block title %}New run — NanoDex{% endblock %}
|
| 4 |
+
{% block body %}
|
| 5 |
+
|
| 6 |
+
<div style="margin-bottom:24px">
|
| 7 |
+
<span class="eyebrow">New training run</span>
|
| 8 |
+
<h1 style="font-size:28px;margin-top:5px">Build a model from nothing</h1>
|
| 9 |
+
</div>
|
| 10 |
+
|
| 11 |
+
<div class="steps">
|
| 12 |
+
<div class="stp on" data-s="1"><div class="n">Step 1</div><div class="t">Size</div></div>
|
| 13 |
+
<div class="stp" data-s="2"><div class="n">Step 2</div><div class="t">Tokens</div></div>
|
| 14 |
+
<div class="stp" data-s="3"><div class="n">Step 3</div><div class="t">Name</div></div>
|
| 15 |
+
<div class="stp" data-s="4"><div class="n">Step 4</div><div class="t">Review</div></div>
|
| 16 |
+
</div>
|
| 17 |
+
|
| 18 |
+
<!-- STEP 1 ---------------------------------------------------------------->
|
| 19 |
+
<div class="panelstep on" data-p="1">
|
| 20 |
+
<h2 style="font-size:19px">How big should it be?</h2>
|
| 21 |
+
<p class="sub" style="margin:6px 0 20px">Every tier is the same architecture —
|
| 22 |
+
a <span class="mono">LlamaForCausalLM</span> decoder-only transformer with
|
| 23 |
+
SiLU MLPs, RMSNorm, rotary embeddings and grouped-query attention — scaled
|
| 24 |
+
down in width and depth. The parameter counts below are totals, embeddings
|
| 25 |
+
included.</p>
|
| 26 |
+
|
| 27 |
+
<div class="picks">
|
| 28 |
+
{% for t in tiers %}
|
| 29 |
+
<button type="button" class="pick {{ 'on' if t.key == '1m' }}" data-tier="{{ t.key }}"
|
| 30 |
+
data-params="{{ t.params }}" data-label="{{ t.label }}">
|
| 31 |
+
<b>{{ t.label }}</b>
|
| 32 |
+
<div class="p-n">{{ "{:,}".format(t.params) }} params</div>
|
| 33 |
+
<div class="p-d">
|
| 34 |
+
{{ t.layers }} layers · {{ t.hidden }} hidden<br>
|
| 35 |
+
{{ t.heads }} heads ({{ t.kv_heads }} KV)<br>
|
| 36 |
+
FFN {{ t.ffn }} · ctx {{ seq_len }}
|
| 37 |
+
</div>
|
| 38 |
+
</button>
|
| 39 |
+
{% endfor %}
|
| 40 |
+
</div>
|
| 41 |
+
|
| 42 |
+
<div style="display:flex;justify-content:flex-end;margin-top:24px">
|
| 43 |
+
<button class="btn primary" data-go="2">Continue →</button>
|
| 44 |
+
</div>
|
| 45 |
+
</div>
|
| 46 |
+
|
| 47 |
+
<!-- STEP 2 ---------------------------------------------------------------->
|
| 48 |
+
<div class="panelstep" data-p="2">
|
| 49 |
+
<h2 style="font-size:19px">How much should it read?</h2>
|
| 50 |
+
<p class="sub" style="margin:6px 0 24px">Training data is
|
| 51 |
+
<a href="https://huggingface.co/datasets/HuggingFaceFW/fineweb-edu" target="_blank"
|
| 52 |
+
rel="noopener" style="color:var(--accent)">fineweb-edu</a> — filtered
|
| 53 |
+
educational web text. More tokens means a sharper model and a longer wait.</p>
|
| 54 |
+
|
| 55 |
+
<div class="card pad-lg">
|
| 56 |
+
<div style="display:flex;justify-content:space-between;align-items:baseline;margin-bottom:16px">
|
| 57 |
+
<label class="fl" style="margin:0">Token budget</label>
|
| 58 |
+
<div class="mono" style="font-size:26px;font-weight:680;color:var(--accent)"
|
| 59 |
+
id="tokLabel">500M</div>
|
| 60 |
+
</div>
|
| 61 |
+
<input type="range" id="tok" min="0" max="{{ stops|length - 1 }}"
|
| 62 |
+
value="4" step="1">
|
| 63 |
+
<div class="mono muted"
|
| 64 |
+
style="display:flex;justify-content:space-between;margin-top:9px;font-size:12px">
|
| 65 |
+
<span>200M</span><span>500M</span><span>1B</span><span>1.5B</span>
|
| 66 |
+
</div>
|
| 67 |
+
<div class="quickpick">
|
| 68 |
+
{% for m in [200, 500, 1000, 1500] %}
|
| 69 |
+
<button type="button" class="btn sm" data-tokens="{{ m }}">
|
| 70 |
+
{{ (m / 1000)|round(1)|string|replace('.0','') ~ 'B' if m >= 1000
|
| 71 |
+
else (m|string ~ 'M') }}</button>
|
| 72 |
+
{% endfor %}
|
| 73 |
+
</div>
|
| 74 |
+
</div>
|
| 75 |
+
|
| 76 |
+
<div class="grid g4" style="margin-top:16px">
|
| 77 |
+
<div class="stat"><span>Parameters</span><b id="e-params">—</b><small id="e-tier"> </small></div>
|
| 78 |
+
<div class="stat"><span>Optimizer steps</span><b id="e-steps">—</b><small>gradient updates</small></div>
|
| 79 |
+
<div class="stat"><span>Estimated time</span><b id="e-eta">—</b><small id="e-cal"> </small></div>
|
| 80 |
+
<div class="stat"><span>Tokens / param</span><b id="e-ratio">—</b><small id="e-chin"> </small></div>
|
| 81 |
+
</div>
|
| 82 |
+
|
| 83 |
+
<p class="hint" id="scaleNote"></p>
|
| 84 |
+
|
| 85 |
+
<div style="display:flex;justify-content:space-between;margin-top:24px">
|
| 86 |
+
<button class="btn ghost" data-go="1">← Back</button>
|
| 87 |
+
<button class="btn primary" data-go="3">Continue →</button>
|
| 88 |
+
</div>
|
| 89 |
+
</div>
|
| 90 |
+
|
| 91 |
+
<!-- STEP 3 ---------------------------------------------------------------->
|
| 92 |
+
<div class="panelstep" data-p="3">
|
| 93 |
+
<h2 style="font-size:19px">Give it a name</h2>
|
| 94 |
+
<p class="sub" style="margin:6px 0 20px">This is what you'll see in your model
|
| 95 |
+
list, and the default repository name when you publish it.</p>
|
| 96 |
+
|
| 97 |
+
<div class="card pad-lg" style="max-width:560px">
|
| 98 |
+
<label class="fl" for="name">Model name</label>
|
| 99 |
+
<input type="text" id="name" placeholder="my-first-lm" maxlength="80" autocomplete="off">
|
| 100 |
+
<p class="hint">Letters, numbers, <span class="mono">-</span>,
|
| 101 |
+
<span class="mono">_</span> and <span class="mono">.</span>. Leave it blank
|
| 102 |
+
and we'll name it after the size and token budget.</p>
|
| 103 |
+
<p class="hint" style="margin-top:14px">Will publish to
|
| 104 |
+
<span class="mono" style="color:var(--fg-2)">{{ user.username }}/<span id="namePreview">…</span></span></p>
|
| 105 |
+
</div>
|
| 106 |
+
|
| 107 |
+
<div style="display:flex;justify-content:space-between;margin-top:24px">
|
| 108 |
+
<button class="btn ghost" data-go="2">← Back</button>
|
| 109 |
+
<button class="btn primary" data-go="4">Review →</button>
|
| 110 |
+
</div>
|
| 111 |
+
</div>
|
| 112 |
+
|
| 113 |
+
<!-- STEP 4 ---------------------------------------------------------------->
|
| 114 |
+
<div class="panelstep" data-p="4">
|
| 115 |
+
<h2 style="font-size:19px">Ready to launch</h2>
|
| 116 |
+
<p class="sub" style="margin:6px 0 20px">Your run joins the shared queue. You
|
| 117 |
+
can watch it — or close the tab and come back; it keeps going either way.</p>
|
| 118 |
+
|
| 119 |
+
<div class="grid g2">
|
| 120 |
+
<div class="card pad-lg">
|
| 121 |
+
<span class="eyebrow">Model</span>
|
| 122 |
+
<div style="margin-top:12px">
|
| 123 |
+
<div class="kv"><span>Name</span><b id="r-name">—</b></div>
|
| 124 |
+
<div class="kv"><span>Size</span><b id="r-tier">—</b></div>
|
| 125 |
+
<div class="kv"><span>Parameters</span><b id="r-params">—</b></div>
|
| 126 |
+
<div class="kv"><span>Architecture</span><b id="r-arch">—</b></div>
|
| 127 |
+
<div class="kv"><span>Vocabulary</span><b>{{ "{:,}".format(vocab) }} BPE</b></div>
|
| 128 |
+
<div class="kv"><span>Context</span><b>{{ seq_len }} tokens</b></div>
|
| 129 |
+
</div>
|
| 130 |
+
</div>
|
| 131 |
+
<div class="card pad-lg">
|
| 132 |
+
<span class="eyebrow">Training</span>
|
| 133 |
+
<div style="margin-top:12px">
|
| 134 |
+
<div class="kv"><span>Dataset</span><b>fineweb-edu</b></div>
|
| 135 |
+
<div class="kv"><span>Token budget</span><b id="r-tok">—</b></div>
|
| 136 |
+
<div class="kv"><span>Optimizer steps</span><b id="r-steps">—</b></div>
|
| 137 |
+
<div class="kv"><span>Optimizer</span><b>AdamW · cosine</b></div>
|
| 138 |
+
<div class="kv"><span>Estimated time</span><b id="r-eta">—</b></div>
|
| 139 |
+
<div class="kv"><span>Queue limit</span><b>{{ max_active }} runs per person</b></div>
|
| 140 |
+
</div>
|
| 141 |
+
</div>
|
| 142 |
+
</div>
|
| 143 |
+
|
| 144 |
+
<div style="display:flex;justify-content:space-between;margin-top:24px;gap:12px;flex-wrap:wrap">
|
| 145 |
+
<button class="btn ghost" data-go="3">← Back</button>
|
| 146 |
+
<button class="btn primary lg" id="launch">Queue this run →</button>
|
| 147 |
+
</div>
|
| 148 |
+
</div>
|
| 149 |
+
|
| 150 |
+
{% endblock %}
|
| 151 |
+
{% block scripts %}
|
| 152 |
+
<script>
|
| 153 |
+
const STOPS = {{ stops | tojson }};
|
| 154 |
+
const state = { tier:'1m', tokens:500, name:'' };
|
| 155 |
+
|
| 156 |
+
function labelTokens(m){
|
| 157 |
+
return m >= 1000 ? String(m/1000).replace(/\.0$/,'') + 'B' : m + 'M';
|
| 158 |
+
}
|
| 159 |
+
let est = null;
|
| 160 |
+
|
| 161 |
+
function showStep(n){
|
| 162 |
+
document.querySelectorAll('.panelstep').forEach(p =>
|
| 163 |
+
p.classList.toggle('on', p.dataset.p === String(n)));
|
| 164 |
+
document.querySelectorAll('.stp').forEach(s => {
|
| 165 |
+
const i = Number(s.dataset.s);
|
| 166 |
+
s.classList.toggle('on', i === n);
|
| 167 |
+
s.classList.toggle('ok', i < n);
|
| 168 |
+
});
|
| 169 |
+
window.scrollTo({top:0, behavior:'smooth'});
|
| 170 |
+
if(n === 4) fillReview();
|
| 171 |
+
}
|
| 172 |
+
|
| 173 |
+
document.querySelectorAll('[data-go]').forEach(b =>
|
| 174 |
+
b.addEventListener('click', () => showStep(Number(b.dataset.go))));
|
| 175 |
+
document.querySelectorAll('.stp').forEach(s =>
|
| 176 |
+
s.addEventListener('click', () => showStep(Number(s.dataset.s))));
|
| 177 |
+
|
| 178 |
+
document.querySelectorAll('.pick[data-tier]').forEach(p =>
|
| 179 |
+
p.addEventListener('click', () => {
|
| 180 |
+
document.querySelectorAll('.pick[data-tier]').forEach(q => q.classList.remove('on'));
|
| 181 |
+
p.classList.add('on');
|
| 182 |
+
state.tier = p.dataset.tier;
|
| 183 |
+
updateEstimate();
|
| 184 |
+
}));
|
| 185 |
+
|
| 186 |
+
const tok = document.getElementById('tok');
|
| 187 |
+
function setTokens(m){
|
| 188 |
+
const i = STOPS.indexOf(m);
|
| 189 |
+
state.tokens = m;
|
| 190 |
+
if(i >= 0) tok.value = i;
|
| 191 |
+
document.getElementById('tokLabel').textContent = labelTokens(m);
|
| 192 |
+
updateEstimate();
|
| 193 |
+
}
|
| 194 |
+
tok.addEventListener('input', () => setTokens(STOPS[Number(tok.value)]));
|
| 195 |
+
document.querySelectorAll('[data-tokens]').forEach(b =>
|
| 196 |
+
b.addEventListener('click', () => setTokens(Number(b.dataset.tokens))));
|
| 197 |
+
|
| 198 |
+
const nameEl = document.getElementById('name');
|
| 199 |
+
nameEl.addEventListener('input', () => {
|
| 200 |
+
state.name = nameEl.value;
|
| 201 |
+
document.getElementById('namePreview').textContent = defaultName();
|
| 202 |
+
});
|
| 203 |
+
|
| 204 |
+
function defaultName(){
|
| 205 |
+
const s = state.name.trim().replace(/[^A-Za-z0-9._-]+/g,'-').replace(/^[-._]+|[-._]+$/g,'');
|
| 206 |
+
if(s) return s.slice(0,80);
|
| 207 |
+
const sel = document.querySelector('.pick.on[data-tier]');
|
| 208 |
+
return (sel ? sel.dataset.label.toLowerCase() : 'nanodex')
|
| 209 |
+
+ '-' + labelTokens(state.tokens).toLowerCase();
|
| 210 |
+
}
|
| 211 |
+
|
| 212 |
+
let estTimer;
|
| 213 |
+
function updateEstimate(){
|
| 214 |
+
clearTimeout(estTimer);
|
| 215 |
+
estTimer = setTimeout(async () => {
|
| 216 |
+
try{
|
| 217 |
+
est = await NX.get(`/api/estimate?tier=${state.tier}&tokens=${state.tokens*1e6}`);
|
| 218 |
+
}catch(e){ return; }
|
| 219 |
+
const sel = document.querySelector('.pick.on[data-tier]');
|
| 220 |
+
document.getElementById('e-params').textContent = Number(est.n_params).toLocaleString();
|
| 221 |
+
document.getElementById('e-tier').textContent = sel ? sel.dataset.label : '';
|
| 222 |
+
document.getElementById('e-steps').textContent = Number(est.steps).toLocaleString();
|
| 223 |
+
document.getElementById('e-eta').textContent = est.eta_human;
|
| 224 |
+
document.getElementById('e-cal').textContent = est.calibrated
|
| 225 |
+
? 'from real runs here' : 'prior — self-calibrates';
|
| 226 |
+
document.getElementById('e-ratio').textContent = Math.round(est.ratio) + ':1';
|
| 227 |
+
document.getElementById('e-chin').textContent =
|
| 228 |
+
est.chinchilla_x.toFixed(1) + '× Chinchilla';
|
| 229 |
+
|
| 230 |
+
const r = est.ratio;
|
| 231 |
+
document.getElementById('scaleNote').textContent = r < 60
|
| 232 |
+
? 'Under ~60 tokens per parameter this model never uses the capacity it has. Either raise the budget or pick a smaller size.'
|
| 233 |
+
: (r < 150
|
| 234 |
+
? 'Workable, but on the thin side for this size — 300:1 or more is where these models get interesting.'
|
| 235 |
+
: (r > 2000
|
| 236 |
+
? 'Very deep into the over-trained regime. Still improves, but the last few hundred million tokens buy little — a larger size would use them better.'
|
| 237 |
+
: 'A good range for this size: well past compute-optimal, and it still finishes in reasonable time.'));
|
| 238 |
+
document.getElementById('namePreview').textContent = defaultName();
|
| 239 |
+
}, 140);
|
| 240 |
+
}
|
| 241 |
+
|
| 242 |
+
function fillReview(){
|
| 243 |
+
const sel = document.querySelector('.pick.on[data-tier]');
|
| 244 |
+
document.getElementById('r-name').textContent = defaultName();
|
| 245 |
+
document.getElementById('r-tier').textContent = sel ? sel.dataset.label : '—';
|
| 246 |
+
document.getElementById('r-params').textContent =
|
| 247 |
+
sel ? Number(sel.dataset.params).toLocaleString() : '—';
|
| 248 |
+
const d = sel ? sel.querySelector('.p-d').textContent.trim().split('·') : [];
|
| 249 |
+
document.getElementById('r-arch').textContent = 'LlamaForCausalLM';
|
| 250 |
+
document.getElementById('r-tok').textContent = labelTokens(state.tokens) + ' tokens';
|
| 251 |
+
document.getElementById('r-steps').textContent =
|
| 252 |
+
est ? Number(est.steps).toLocaleString() : '—';
|
| 253 |
+
document.getElementById('r-eta').textContent = est ? est.eta_human : '—';
|
| 254 |
+
}
|
| 255 |
+
|
| 256 |
+
document.getElementById('launch').addEventListener('click', async (ev) => {
|
| 257 |
+
const b = ev.currentTarget;
|
| 258 |
+
b.disabled = true; b.textContent = 'Queueing…';
|
| 259 |
+
try{
|
| 260 |
+
const r = await NX.post('/api/jobs', {
|
| 261 |
+
tier: state.tier, tokens: state.tokens * 1e6, name: state.name
|
| 262 |
+
});
|
| 263 |
+
location.href = '/models/' + r.id;
|
| 264 |
+
}catch(e){
|
| 265 |
+
NX.toast(NX.esc(e.message), 'err');
|
| 266 |
+
b.disabled = false; b.innerHTML = 'Queue this run →';
|
| 267 |
+
}
|
| 268 |
+
});
|
| 269 |
+
|
| 270 |
+
setTokens(state.tokens);
|
| 271 |
+
document.getElementById('namePreview').textContent = defaultName();
|
| 272 |
+
</script>
|
| 273 |
+
{% endblock %}
|
web/templates/user.html
ADDED
|
@@ -0,0 +1,100 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{% extends "base.html" %}
|
| 2 |
+
{% set nav = 'explore' %}
|
| 3 |
+
{% block title %}{{ profile.display_name }} — NanoDex{% endblock %}
|
| 4 |
+
{% block body %}
|
| 5 |
+
|
| 6 |
+
<div class="card pad-lg" style="display:flex;gap:20px;align-items:center;flex-wrap:wrap">
|
| 7 |
+
{% if profile.avatar_url %}
|
| 8 |
+
<img src="{{ profile.avatar_url }}" alt="" style="width:72px;height:72px;
|
| 9 |
+
border-radius:50%;border:1px solid var(--line-2);object-fit:cover">
|
| 10 |
+
{% else %}
|
| 11 |
+
<div style="width:72px;height:72px;border-radius:50%;background:var(--panel-2);
|
| 12 |
+
border:1px solid var(--line-2)"></div>
|
| 13 |
+
{% endif %}
|
| 14 |
+
<div style="flex:1;min-width:200px">
|
| 15 |
+
<div style="display:flex;align-items:center;gap:10px;flex-wrap:wrap">
|
| 16 |
+
<h1 style="font-size:24px">{{ profile.display_name }}</h1>
|
| 17 |
+
{% if is_me %}<span class="chip done"><i></i>you</span>{% endif %}
|
| 18 |
+
</div>
|
| 19 |
+
<p class="sub" style="margin-top:3px">
|
| 20 |
+
<a href="https://huggingface.co/{{ profile.username }}" target="_blank"
|
| 21 |
+
rel="noopener" class="mono" style="color:var(--fg-2)">@{{ profile.username }}</a>
|
| 22 |
+
· training here since
|
| 23 |
+
<span id="since">{{ profile.first_seen }}</span>
|
| 24 |
+
</p>
|
| 25 |
+
</div>
|
| 26 |
+
<div style="display:flex;gap:8px">
|
| 27 |
+
{% if is_me %}<a class="btn" href="/train">+ New run</a>{% endif %}
|
| 28 |
+
<a class="btn ghost" href="/explore">Explore all →</a>
|
| 29 |
+
</div>
|
| 30 |
+
</div>
|
| 31 |
+
|
| 32 |
+
<div class="grid g4" style="margin-top:16px">
|
| 33 |
+
<div class="stat"><span>Models</span><b>{{ profile.done }}</b>
|
| 34 |
+
<small>{{ profile.runs }} run{{ 's' if profile.runs != 1 }} total{{ ', ' ~ profile.active ~ ' in flight' if profile.active }}</small></div>
|
| 35 |
+
<div class="stat"><span>Tokens trained</span>
|
| 36 |
+
<b>{{ "%.0fM"|format(profile.tokens / 1000000) if profile.tokens < 1000000000 else "%.2fB"|format(profile.tokens / 1000000000) }}</b>
|
| 37 |
+
<small>of fineweb-edu</small></div>
|
| 38 |
+
<div class="stat"><span>Parameters created</span>
|
| 39 |
+
<b>{{ "{:,}".format(profile.params) }}</b><small>across finished models</small></div>
|
| 40 |
+
<div class="stat"><span>Best loss</span>
|
| 41 |
+
<b>{{ "%.4f"|format(profile.best_loss) if profile.best_loss else "—" }}</b>
|
| 42 |
+
<small>{{ profile.published }} published to the Hub</small></div>
|
| 43 |
+
</div>
|
| 44 |
+
|
| 45 |
+
<section id="activeSec" style="margin-top:32px;display:none">
|
| 46 |
+
<div class="sect-h"><h2>Training now</h2></div>
|
| 47 |
+
<div class="runs" id="active"></div>
|
| 48 |
+
</section>
|
| 49 |
+
|
| 50 |
+
<section style="margin-top:32px">
|
| 51 |
+
<div class="sect-h">
|
| 52 |
+
<h2>Models</h2>
|
| 53 |
+
<span class="sub">open any of them to generate text</span>
|
| 54 |
+
</div>
|
| 55 |
+
<div id="models"></div>
|
| 56 |
+
</section>
|
| 57 |
+
|
| 58 |
+
{% endblock %}
|
| 59 |
+
{% block scripts %}
|
| 60 |
+
<script>
|
| 61 |
+
const USERNAME = {{ profile.username | tojson }};
|
| 62 |
+
const SINCE = {{ profile.first_seen | tojson }};
|
| 63 |
+
document.getElementById('since').textContent = SINCE
|
| 64 |
+
? new Date(SINCE * 1000).toLocaleDateString(undefined,
|
| 65 |
+
{year:'numeric', month:'long'})
|
| 66 |
+
: '—';
|
| 67 |
+
|
| 68 |
+
function card(m){
|
| 69 |
+
return `<a class="run" href="/models/${m.id}">
|
| 70 |
+
<div class="run-top">
|
| 71 |
+
<h4>${NX.esc(m.name)}</h4>
|
| 72 |
+
${NX.chip(m.status)}
|
| 73 |
+
</div>
|
| 74 |
+
<div class="run-meta">
|
| 75 |
+
<span class="mono">${NX.esc(m.tier_label)}</span>
|
| 76 |
+
<span>${Number(m.n_params).toLocaleString()} params</span>
|
| 77 |
+
<span><b>${NX.fmtTokens(m.tokens_seen)}</b> tokens</span>
|
| 78 |
+
${m.best_loss ? `<span>loss <b>${m.best_loss.toFixed(4)}</b></span>` : ''}
|
| 79 |
+
${m.pushed_repo
|
| 80 |
+
? `<span class="mono" style="color:var(--accent)">${NX.esc(m.pushed_repo)}</span>`
|
| 81 |
+
: '<span class="muted">not published</span>'}
|
| 82 |
+
<span class="muted">${NX.ago(m.finished_at || m.created_at)}</span>
|
| 83 |
+
</div>
|
| 84 |
+
</a>`;
|
| 85 |
+
}
|
| 86 |
+
|
| 87 |
+
async function refresh(){
|
| 88 |
+
const d = await NX.get('/api/users/' + encodeURIComponent(USERNAME));
|
| 89 |
+
|
| 90 |
+
const sec = document.getElementById('activeSec');
|
| 91 |
+
sec.style.display = d.active.length ? 'block' : 'none';
|
| 92 |
+
document.getElementById('active').innerHTML = d.active.map(NX.runRow).join('');
|
| 93 |
+
|
| 94 |
+
document.getElementById('models').innerHTML = d.models.length
|
| 95 |
+
? `<div class="runs">${d.models.map(card).join('')}</div>`
|
| 96 |
+
: `<div class="empty">No finished models yet.</div>`;
|
| 97 |
+
}
|
| 98 |
+
NX.poll(refresh, 6000);
|
| 99 |
+
</script>
|
| 100 |
+
{% endblock %}
|