Spaces:
Sleeping
Sleeping
Add training worker
Browse files- app/services/training.py +130 -0
app/services/training.py
ADDED
|
@@ -0,0 +1,130 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Background training worker.
|
| 2 |
+
|
| 3 |
+
Runs *real* gradient-descent training of a small 3-layer MLP with NumPy
|
| 4 |
+
(no GPU required). Each epoch persists loss to the DB so clients can stream
|
| 5 |
+
metrics. To fine-tune real models, replace `_run` with a TRL/transformers
|
| 6 |
+
training call (see README "Fine-tuning real models").
|
| 7 |
+
"""
|
| 8 |
+
import datetime
|
| 9 |
+
import threading
|
| 10 |
+
|
| 11 |
+
import numpy as np
|
| 12 |
+
|
| 13 |
+
from .. import models as orm
|
| 14 |
+
from ..database import SessionLocal
|
| 15 |
+
|
| 16 |
+
_workers = {}
|
| 17 |
+
_cancel = set()
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
def _forward_backward(X, y, W1, b1, W2, b2):
|
| 21 |
+
z = X @ W1 + b1
|
| 22 |
+
h = np.tanh(z)
|
| 23 |
+
logits = h @ W2 + b2
|
| 24 |
+
ex = np.exp(logits - logits.max(axis=1, keepdims=True))
|
| 25 |
+
p = ex / ex.sum(axis=1, keepdims=True)
|
| 26 |
+
loss = -np.log(p[np.arange(len(y)), y]).mean()
|
| 27 |
+
|
| 28 |
+
dp = p.copy()
|
| 29 |
+
dp[np.arange(len(y)), y] -= 1.0
|
| 30 |
+
dp /= len(y)
|
| 31 |
+
|
| 32 |
+
dW2 = h.T @ dp
|
| 33 |
+
db2 = dp.sum(0)
|
| 34 |
+
dh = dp @ W2.T
|
| 35 |
+
dz = dh * (1.0 - np.tanh(z) ** 2)
|
| 36 |
+
dW1 = X.T @ dz
|
| 37 |
+
db1 = dz.sum(0)
|
| 38 |
+
|
| 39 |
+
return loss, dW1, db1, dW2, db2
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
def _run(job_id: str):
|
| 43 |
+
db = SessionLocal()
|
| 44 |
+
try:
|
| 45 |
+
job = db.get(orm.TrainingJob, job_id)
|
| 46 |
+
if job is None:
|
| 47 |
+
return
|
| 48 |
+
cfg = job.config or {}
|
| 49 |
+
rng = np.random.default_rng(int(cfg.get("seed", 42)))
|
| 50 |
+
n, d, k = 600, 8, 4
|
| 51 |
+
X = rng.normal(size=(n, d))
|
| 52 |
+
true_W = rng.normal(size=(d, d))
|
| 53 |
+
y = (np.argmax(X @ true_W + 0.4 * rng.normal(size=(n, d)), axis=1) % k).astype(int)
|
| 54 |
+
|
| 55 |
+
W1 = rng.normal(size=(d, 32)) * 0.1
|
| 56 |
+
b1 = np.zeros(32)
|
| 57 |
+
W2 = rng.normal(size=(32, k)) * 0.1
|
| 58 |
+
b2 = np.zeros(k)
|
| 59 |
+
|
| 60 |
+
lr = float(cfg.get("lr", 0.05))
|
| 61 |
+
epochs = int(cfg.get("epochs", 20))
|
| 62 |
+
bs = int(cfg.get("batch_size", 32))
|
| 63 |
+
|
| 64 |
+
job.status = "running"
|
| 65 |
+
job.started_at = datetime.datetime.utcnow()
|
| 66 |
+
loss_history = (job.metrics or {}).get("loss") if isinstance(job.metrics, dict) else None
|
| 67 |
+
loss_history = list(loss_history) if loss_history else []
|
| 68 |
+
db.commit()
|
| 69 |
+
|
| 70 |
+
for ep in range(epochs):
|
| 71 |
+
if job_id in _cancel:
|
| 72 |
+
job.status = "cancelled"
|
| 73 |
+
job.finished_at = datetime.datetime.utcnow()
|
| 74 |
+
db.commit()
|
| 75 |
+
return
|
| 76 |
+
|
| 77 |
+
perm = rng.permutation(n)
|
| 78 |
+
tot, nb = 0.0, 0
|
| 79 |
+
for i in range(0, n, bs):
|
| 80 |
+
idx = perm[i:i + bs]
|
| 81 |
+
loss, dW1, db1, dW2, db2 = _forward_backward(X[idx], y[idx], W1, b1, W2, b2)
|
| 82 |
+
W2 -= lr * dW2
|
| 83 |
+
b2 -= lr * db2
|
| 84 |
+
W1 -= lr * dW1
|
| 85 |
+
b1 -= lr * db1
|
| 86 |
+
tot += loss
|
| 87 |
+
nb += 1
|
| 88 |
+
|
| 89 |
+
avg = float(tot / nb)
|
| 90 |
+
loss_history.append(avg)
|
| 91 |
+
job.current_epoch = ep + 1
|
| 92 |
+
job.loss = avg
|
| 93 |
+
job.metrics = {"loss": loss_history[-200:], "lr": lr, "batch_size": bs}
|
| 94 |
+
db.commit()
|
| 95 |
+
|
| 96 |
+
h = np.tanh(X @ W1 + b1)
|
| 97 |
+
preds = np.argmax(h @ W2 + b2, axis=1)
|
| 98 |
+
acc = float((preds == y).mean())
|
| 99 |
+
job.status = "completed"
|
| 100 |
+
job.finished_at = datetime.datetime.utcnow()
|
| 101 |
+
job.metrics = {"loss": loss_history[-200:], "accuracy": acc}
|
| 102 |
+
db.commit()
|
| 103 |
+
except Exception as e: # noqa: BLE001
|
| 104 |
+
try:
|
| 105 |
+
job.status = "failed"
|
| 106 |
+
job.error = str(e)
|
| 107 |
+
job.finished_at = datetime.datetime.utcnow()
|
| 108 |
+
db.commit()
|
| 109 |
+
except Exception:
|
| 110 |
+
pass
|
| 111 |
+
finally:
|
| 112 |
+
db.close()
|
| 113 |
+
|
| 114 |
+
|
| 115 |
+
def launch(job_id: str) -> bool:
|
| 116 |
+
"""Start a training thread for the job; returns False if already running."""
|
| 117 |
+
if job_id in _workers and _workers[job_id].is_alive():
|
| 118 |
+
return False
|
| 119 |
+
t = threading.Thread(target=_run, args=(job_id,), daemon=True)
|
| 120 |
+
_workers[job_id] = t
|
| 121 |
+
t.start()
|
| 122 |
+
return True
|
| 123 |
+
|
| 124 |
+
|
| 125 |
+
def cancel(job_id: str) -> None:
|
| 126 |
+
_cancel.add(job_id)
|
| 127 |
+
|
| 128 |
+
|
| 129 |
+
def is_running(job_id: str) -> bool:
|
| 130 |
+
return job_id in _workers and _workers[job_id].is_alive()
|