Taimwe commited on
Commit
b89a559
·
verified ·
1 Parent(s): 15de297

Add training worker

Browse files
Files changed (1) hide show
  1. app/services/training.py +130 -0
app/services/training.py ADDED
@@ -0,0 +1,130 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Background training worker.
2
+
3
+ Runs *real* gradient-descent training of a small 3-layer MLP with NumPy
4
+ (no GPU required). Each epoch persists loss to the DB so clients can stream
5
+ metrics. To fine-tune real models, replace `_run` with a TRL/transformers
6
+ training call (see README "Fine-tuning real models").
7
+ """
8
+ import datetime
9
+ import threading
10
+
11
+ import numpy as np
12
+
13
+ from .. import models as orm
14
+ from ..database import SessionLocal
15
+
16
+ _workers = {}
17
+ _cancel = set()
18
+
19
+
20
+ def _forward_backward(X, y, W1, b1, W2, b2):
21
+ z = X @ W1 + b1
22
+ h = np.tanh(z)
23
+ logits = h @ W2 + b2
24
+ ex = np.exp(logits - logits.max(axis=1, keepdims=True))
25
+ p = ex / ex.sum(axis=1, keepdims=True)
26
+ loss = -np.log(p[np.arange(len(y)), y]).mean()
27
+
28
+ dp = p.copy()
29
+ dp[np.arange(len(y)), y] -= 1.0
30
+ dp /= len(y)
31
+
32
+ dW2 = h.T @ dp
33
+ db2 = dp.sum(0)
34
+ dh = dp @ W2.T
35
+ dz = dh * (1.0 - np.tanh(z) ** 2)
36
+ dW1 = X.T @ dz
37
+ db1 = dz.sum(0)
38
+
39
+ return loss, dW1, db1, dW2, db2
40
+
41
+
42
+ def _run(job_id: str):
43
+ db = SessionLocal()
44
+ try:
45
+ job = db.get(orm.TrainingJob, job_id)
46
+ if job is None:
47
+ return
48
+ cfg = job.config or {}
49
+ rng = np.random.default_rng(int(cfg.get("seed", 42)))
50
+ n, d, k = 600, 8, 4
51
+ X = rng.normal(size=(n, d))
52
+ true_W = rng.normal(size=(d, d))
53
+ y = (np.argmax(X @ true_W + 0.4 * rng.normal(size=(n, d)), axis=1) % k).astype(int)
54
+
55
+ W1 = rng.normal(size=(d, 32)) * 0.1
56
+ b1 = np.zeros(32)
57
+ W2 = rng.normal(size=(32, k)) * 0.1
58
+ b2 = np.zeros(k)
59
+
60
+ lr = float(cfg.get("lr", 0.05))
61
+ epochs = int(cfg.get("epochs", 20))
62
+ bs = int(cfg.get("batch_size", 32))
63
+
64
+ job.status = "running"
65
+ job.started_at = datetime.datetime.utcnow()
66
+ loss_history = (job.metrics or {}).get("loss") if isinstance(job.metrics, dict) else None
67
+ loss_history = list(loss_history) if loss_history else []
68
+ db.commit()
69
+
70
+ for ep in range(epochs):
71
+ if job_id in _cancel:
72
+ job.status = "cancelled"
73
+ job.finished_at = datetime.datetime.utcnow()
74
+ db.commit()
75
+ return
76
+
77
+ perm = rng.permutation(n)
78
+ tot, nb = 0.0, 0
79
+ for i in range(0, n, bs):
80
+ idx = perm[i:i + bs]
81
+ loss, dW1, db1, dW2, db2 = _forward_backward(X[idx], y[idx], W1, b1, W2, b2)
82
+ W2 -= lr * dW2
83
+ b2 -= lr * db2
84
+ W1 -= lr * dW1
85
+ b1 -= lr * db1
86
+ tot += loss
87
+ nb += 1
88
+
89
+ avg = float(tot / nb)
90
+ loss_history.append(avg)
91
+ job.current_epoch = ep + 1
92
+ job.loss = avg
93
+ job.metrics = {"loss": loss_history[-200:], "lr": lr, "batch_size": bs}
94
+ db.commit()
95
+
96
+ h = np.tanh(X @ W1 + b1)
97
+ preds = np.argmax(h @ W2 + b2, axis=1)
98
+ acc = float((preds == y).mean())
99
+ job.status = "completed"
100
+ job.finished_at = datetime.datetime.utcnow()
101
+ job.metrics = {"loss": loss_history[-200:], "accuracy": acc}
102
+ db.commit()
103
+ except Exception as e: # noqa: BLE001
104
+ try:
105
+ job.status = "failed"
106
+ job.error = str(e)
107
+ job.finished_at = datetime.datetime.utcnow()
108
+ db.commit()
109
+ except Exception:
110
+ pass
111
+ finally:
112
+ db.close()
113
+
114
+
115
+ def launch(job_id: str) -> bool:
116
+ """Start a training thread for the job; returns False if already running."""
117
+ if job_id in _workers and _workers[job_id].is_alive():
118
+ return False
119
+ t = threading.Thread(target=_run, args=(job_id,), daemon=True)
120
+ _workers[job_id] = t
121
+ t.start()
122
+ return True
123
+
124
+
125
+ def cancel(job_id: str) -> None:
126
+ _cancel.add(job_id)
127
+
128
+
129
+ def is_running(job_id: str) -> bool:
130
+ return job_id in _workers and _workers[job_id].is_alive()