Z User commited on
Commit
d59cbfd
Β·
1 Parent(s): 91dede3

init: RigoChat-7B worker node 1

Browse files
Files changed (3) hide show
  1. Dockerfile +34 -0
  2. package.json +10 -0
  3. server.js +232 -0
Dockerfile ADDED
@@ -0,0 +1,34 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ FROM node:20-slim
2
+
3
+ # ── System deps ──────────────────────────────────────────────────────────────
4
+ RUN apt-get update && apt-get install -y --no-install-recommends \
5
+ make g++ python3 curl wget ca-certificates \
6
+ && rm -rf /var/lib/apt/lists/*
7
+
8
+ # ── CPU Performance ──────────────────────────────────────────────────────────
9
+ ENV NODE_OPTIONS="--max-old-space-size=6144"
10
+ ENV UV_THREADPOOL_SIZE=16
11
+
12
+ WORKDIR /app
13
+
14
+ # ── Install deps ─────────────────────────────────────────────────────────────
15
+ COPY package.json ./
16
+ RUN npm install --production 2>&1 | tail -5 || \
17
+ npm install --legacy-peer-deps 2>&1 | tail -5 || true
18
+
19
+ # ── Copy app ─────────────────────────────────────────────────────────────────
20
+ COPY . .
21
+
22
+ # ── Model cache dir ──────────────────────────────────────────────────────────
23
+ RUN mkdir -p /app/models /tmp/worker-cache
24
+
25
+ ENV PORT=7860
26
+ ENV NODE_ENV=production
27
+ ENV MODEL_DIR=/app/models
28
+
29
+ EXPOSE 7860
30
+
31
+ HEALTHCHECK --interval=30s --timeout=10s --start-period=180s --retries=5 \
32
+ CMD curl -f http://localhost:7860/health || exit 1
33
+
34
+ CMD ["node", "server.js"]
package.json ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "rigochat-worker",
3
+ "version": "1.0.0",
4
+ "description": "RigoChat-7B distributed inference worker",
5
+ "type": "module",
6
+ "main": "server.js",
7
+ "dependencies": {
8
+ "node-llama-cpp": "^3.4.0"
9
+ }
10
+ }
server.js ADDED
@@ -0,0 +1,232 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ /**
2
+ * RigoChat Worker β€” Distributed Inference Node
3
+ * ================================================
4
+ * Lightweight HF Space that loads RigoChat-7B-v2 and exposes
5
+ * an /inference endpoint. Multiple workers can be combined
6
+ * via Promise.any for faster responses.
7
+ */
8
+
9
+ import http from 'http';
10
+ import { existsSync, mkdirSync, readdirSync } from 'fs';
11
+ import path from 'path';
12
+
13
+ // ── Config ──────────────────────────────────────────────────────────────────
14
+ const PORT = parseInt(process.env.PORT ?? '7860');
15
+ const MODEL_DIR = process.env.MODEL_DIR ?? './models';
16
+ const MODEL_REPO = process.env.MODEL_REPO ?? 'IIC/RigoChat-7b-v2-GGUF';
17
+ const MODEL_FILE = process.env.MODEL_FILE ?? 'rigochat-7b-v2-Q4_K_M.gguf';
18
+ const CTX_SIZE = parseInt(process.env.CTX_SIZE ?? '4096');
19
+ const WORKER_ID = process.env.SPACE_ID ?? `worker-${Math.random().toString(36).slice(2, 6)}`;
20
+ const AUTH_KEY = process.env.WORKER_KEY ?? 'zelin-cluster'; // Simple auth
21
+
22
+ // ── State ───────────────────────────────────────────────────────────────────
23
+ let _model = null;
24
+ let _ready = false;
25
+ let _loading = false;
26
+ let _stats = { requests: 0, totalMs: 0, errors: 0, avgMs: 0 };
27
+
28
+ // ── Load Model ──────────────────────────────────────────────────────────────
29
+ async function loadModel() {
30
+ if (_ready || _loading) return;
31
+ _loading = true;
32
+ const start = Date.now();
33
+
34
+ try {
35
+ console.log(`[Worker] Loading ${MODEL_FILE}...`);
36
+ mkdirSync(MODEL_DIR, { recursive: true });
37
+
38
+ const { getLlama } = await import('node-llama-cpp');
39
+ const llama = await getLlama();
40
+
41
+ const modelPath = path.join(MODEL_DIR, MODEL_FILE);
42
+
43
+ // Download if needed
44
+ if (!existsSync(modelPath)) {
45
+ // Check for HF-prefixed names
46
+ const files = readdirSync(MODEL_DIR).filter(f => f.endsWith('.gguf'));
47
+ const found = files.find(f => f.includes('rigochat') || f.includes('RigoChat'));
48
+
49
+ if (!found) {
50
+ console.log(`[Worker] Downloading ${MODEL_FILE} (~1GB)...`);
51
+ const { createModelDownloader } = await import('node-llama-cpp');
52
+ const dl = await createModelDownloader({
53
+ modelUri: `hf:${MODEL_REPO}/${MODEL_FILE}`,
54
+ dirPath: MODEL_DIR,
55
+ onProgress: ({ downloadedSize, totalSize }) => {
56
+ const pct = totalSize ? Math.round(downloadedSize / totalSize * 100) : '?';
57
+ process.stdout.write(`\r[Worker] Downloading... ${pct}%`);
58
+ },
59
+ });
60
+ await dl.download();
61
+ console.log('\n[Worker] Download complete βœ…');
62
+ }
63
+ }
64
+
65
+ // Find actual model file
66
+ let actualPath = modelPath;
67
+ if (!existsSync(modelPath)) {
68
+ const files = readdirSync(MODEL_DIR).filter(f => f.endsWith('.gguf'));
69
+ const found = files.find(f => f.includes('rigochat') || f.includes('RigoChat'));
70
+ if (found) actualPath = path.join(MODEL_DIR, found);
71
+ }
72
+
73
+ console.log('[Worker] Loading model into memory...');
74
+ _model = await llama.loadModel({ modelPath: actualPath, gpuLayers: 0 });
75
+ _ready = true;
76
+ _loading = false;
77
+
78
+ const elapsed = ((Date.now() - start) / 1000).toFixed(1);
79
+ console.log(`[Worker] βœ… Ready in ${elapsed}s β€” ${WORKER_ID}`);
80
+ } catch (err) {
81
+ _loading = false;
82
+ console.error('[Worker] Load error:', err.message);
83
+ }
84
+ }
85
+
86
+ // ── Inference ───────────────────────────────────────────────────────────────
87
+ async function infer(messages, maxTokens = 300, temperature = 0.7) {
88
+ if (!_model) throw new Error('Model not loaded');
89
+
90
+ const { LlamaChatSession } = await import('node-llama-cpp');
91
+ const ctx = await _model.createContext({ contextSize: CTX_SIZE });
92
+
93
+ const systemMsg = messages.find(m => m.role === 'system')?.content ?? '';
94
+ const stylePrefix = 'Responde en espaΓ±ol casual argentino. MΓ‘x 2 lΓ­neas. Sin mayΓΊsculas al inicio. Sin punto final.\n\n';
95
+ const sysFinal = systemMsg.includes('espaΓ±ol') ? systemMsg : stylePrefix + systemMsg;
96
+
97
+ const session = new LlamaChatSession({
98
+ contextSequence: ctx.getSequence(),
99
+ systemPrompt: sysFinal,
100
+ });
101
+
102
+ const userMsgs = messages.filter(m => m.role !== 'system');
103
+ const lastUser = userMsgs[userMsgs.length - 1];
104
+
105
+ // Feed history
106
+ for (const msg of userMsgs.slice(0, -1)) {
107
+ await session.prompt(msg.content ?? '', { maxTokens: 1 }).catch(() => {});
108
+ }
109
+
110
+ let result = '';
111
+ if (lastUser) {
112
+ result = await session.prompt(lastUser.content ?? '', {
113
+ maxTokens,
114
+ temperature,
115
+ topP: 0.9,
116
+ topK: 40,
117
+ minP: 0.05,
118
+ repeatPenalty: { penalty: 1.35, lastTokens: 96, frequencyPenalty: 0.1, presencePenalty: 0.05 },
119
+ });
120
+ }
121
+
122
+ session.dispose?.();
123
+ ctx.dispose?.();
124
+
125
+ return result.trim();
126
+ }
127
+
128
+ // ── HTTP Server ─────────────────────────────────────────────────────────────
129
+ const server = http.createServer(async (req, res) => {
130
+ // CORS
131
+ res.setHeader('Access-Control-Allow-Origin', '*');
132
+ res.setHeader('Access-Control-Allow-Methods', 'GET, POST, OPTIONS');
133
+ res.setHeader('Access-Control-Allow-Headers', 'Content-Type, Authorization');
134
+
135
+ if (req.method === 'OPTIONS') { res.writeHead(204); res.end(); return; }
136
+
137
+ const url = new URL(req.url, `http://localhost:${PORT}`);
138
+
139
+ // Health check
140
+ if (url.pathname === '/health') {
141
+ res.writeHead(200, { 'Content-Type': 'application/json' });
142
+ res.end(JSON.stringify({
143
+ status: _ready ? 'ready' : (_loading ? 'loading' : 'error'),
144
+ worker: WORKER_ID,
145
+ model: MODEL_FILE,
146
+ uptime: process.uptime(),
147
+ memory: Math.round(process.memoryUsage().heapUsed / 1024 / 1024) + 'MB',
148
+ stats: _stats,
149
+ }));
150
+ return;
151
+ }
152
+
153
+ // Inference endpoint
154
+ if (url.pathname === '/inference' && req.method === 'POST') {
155
+ // Simple auth
156
+ const auth = req.headers['authorization'];
157
+ if (auth !== `Bearer ${AUTH_KEY}`) {
158
+ res.writeHead(401, { 'Content-Type': 'application/json' });
159
+ res.end(JSON.stringify({ error: 'Unauthorized' }));
160
+ return;
161
+ }
162
+
163
+ if (!_ready) {
164
+ res.writeHead(503, { 'Content-Type': 'application/json' });
165
+ res.end(JSON.stringify({ error: 'Model not ready', status: _loading ? 'loading' : 'error' }));
166
+ return;
167
+ }
168
+
169
+ try {
170
+ const body = await new Promise((resolve, reject) => {
171
+ let data = '';
172
+ req.on('data', c => data += c);
173
+ req.on('end', () => resolve(JSON.parse(data)));
174
+ req.on('error', reject);
175
+ setTimeout(() => reject(new Error('Timeout')), 30000);
176
+ });
177
+
178
+ const { messages, maxTokens = 300, temperature = 0.7 } = body;
179
+ if (!messages?.length) {
180
+ res.writeHead(400, { 'Content-Type': 'application/json' });
181
+ res.end(JSON.stringify({ error: 'messages required' }));
182
+ return;
183
+ }
184
+
185
+ const start = Date.now();
186
+ const result = await infer(messages, maxTokens, temperature);
187
+ const ms = Date.now() - start;
188
+
189
+ _stats.requests++;
190
+ _stats.totalMs += ms;
191
+ _stats.avgMs = Math.round(_stats.totalMs / _stats.requests);
192
+
193
+ res.writeHead(200, { 'Content-Type': 'application/json' });
194
+ res.end(JSON.stringify({
195
+ result,
196
+ worker: WORKER_ID,
197
+ latencyMs: ms,
198
+ tokens: result.split(/\s+/).length,
199
+ }));
200
+ } catch (err) {
201
+ _stats.errors++;
202
+ res.writeHead(500, { 'Content-Type': 'application/json' });
203
+ res.end(JSON.stringify({ error: err.message, worker: WORKER_ID }));
204
+ }
205
+ return;
206
+ }
207
+
208
+ // Root
209
+ if (url.pathname === '/') {
210
+ res.writeHead(200, { 'Content-Type': 'application/json' });
211
+ res.end(JSON.stringify({
212
+ name: 'rigochat-worker',
213
+ version: '1.0.0',
214
+ status: _ready ? 'ready' : (_loading ? 'loading' : 'error'),
215
+ worker: WORKER_ID,
216
+ model: MODEL_FILE,
217
+ endpoints: ['/health', '/inference'],
218
+ }));
219
+ return;
220
+ }
221
+
222
+ res.writeHead(404);
223
+ res.end('Not found');
224
+ });
225
+
226
+ // ── Start ───────────────────────────────────────────────────────────────────
227
+ server.listen(PORT, '0.0.0.0', () => {
228
+ console.log(`[Worker] HTTP server on port ${PORT}`);
229
+ console.log(`[Worker] Worker ID: ${WORKER_ID}`);
230
+ console.log(`[Worker] Loading model in background...`);
231
+ loadModel();
232
+ });