File size: 4,336 Bytes
2a49d9a
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
"""One worker thread per GPU, pulling jobs off the shared queue."""

import threading
import time
import traceback

from . import db
from .trainer import run_training

_cancel = set()
_cancel_lock = threading.Lock()
_workers = []


def request_cancel(job_id):
    with _cancel_lock:
        _cancel.add(job_id)
    j = db.get_job(job_id)
    if j and j["status"] == "queued":
        db.update_job(job_id, status="cancelled", finished_at=time.time())
        return True
    return bool(j)


def _is_cancelled(job_id):
    with _cancel_lock:
        return job_id in _cancel


_devices_cache = None


def devices():
    """Never raises and never blocks twice — CUDA probing is done once."""
    global _devices_cache
    if _devices_cache is not None:
        return _devices_cache
    found = ["cpu"]
    try:
        import torch
        if torch.cuda.is_available() and torch.cuda.device_count() > 0:
            found = [f"cuda:{i}" for i in range(torch.cuda.device_count())]
    except Exception as exc:
        print(f"[worker] no CUDA ({exc!r}) — falling back to CPU", flush=True)
    _devices_cache = found
    return found


def _loop(gpu_index, device, tokenizer_dir):
    while True:
        try:
            job = db.claim_next_queued(gpu_index)
        except Exception:
            job = None
        if not job:
            time.sleep(2)
            continue

        jid = job["id"]

        def log(msg, _jid=jid, _dev=device):
            print(f"[{_dev}][{_jid}] {msg}", flush=True)
            try:
                db.add_log(_jid, msg)
            except Exception:
                pass

        # Don't start on a barely-filled cache, or a 300M-token run would
        # loop over the same few million tokens hundreds of times.
        try:
            from . import data as ndata
            need = min(int(job["target_tokens"]), ndata.TARGET_TOKENS)
            last_note = 0.0
            while ndata.available_tokens() < need and ndata.state()["status"] == "building":
                if time.time() - last_note > 60:
                    log(f"waiting for the fineweb-edu cache to reach "
                        f"{need / 1e6:.0f}M tokens "
                        f"({ndata.available_tokens() / 1e6:.0f}M ready so far — "
                        f"this is a one-time cost per container)")
                    last_note = time.time()
                if _is_cancelled(jid):
                    break
                time.sleep(5)
        except Exception:
            pass

        if _is_cancelled(jid):
            db.update_job(jid, status="cancelled", finished_at=time.time())
            with _cancel_lock:
                _cancel.discard(jid)
            continue

        log(f"starting on {device}")
        try:
            run_training(job, device, tokenizer_dir, lambda: _is_cancelled(jid), log)
        except Exception as exc:
            tb = traceback.format_exc(limit=6)
            print(tb, flush=True)
            log(f"FAILED: {exc!r}")
            db.update_job(jid, status="failed", error=str(exc)[:500],
                          finished_at=time.time())
        finally:
            with _cancel_lock:
                _cancel.discard(jid)
            try:
                import torch
                if device.startswith("cuda"):
                    torch.cuda.empty_cache()
            except Exception:
                pass


def start_workers(tokenizer_dir):
    global _workers
    if _workers:
        return _workers
    for i, dev in enumerate(devices()):
        if dev.startswith("cuda"):
            try:
                import torch
                name = torch.cuda.get_device_name(i)
                gb = torch.cuda.get_device_properties(i).total_memory / 1024**3
                print(f"[worker] {dev}: {name} ({gb:.0f} GB)", flush=True)
            except Exception:
                pass
        t = threading.Thread(target=_loop, args=(i, dev, tokenizer_dir),
                             name=f"worker-{dev}", daemon=True)
        t.start()
        _workers.append(t)
        print(f"[worker] started on {dev}", flush=True)
    return _workers


def worker_info():
    devs = devices()
    running = {j["gpu_index"]: j for j in db.list_jobs(status=["running"], limit=64)}
    return [{"index": i, "device": d, "job": running.get(i)} for i, d in enumerate(devs)]