Z User commited on
Commit Β·
d59cbfd
1
Parent(s): 91dede3
init: RigoChat-7B worker node 1
Browse files- Dockerfile +34 -0
- package.json +10 -0
- server.js +232 -0
Dockerfile
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
FROM node:20-slim
|
| 2 |
+
|
| 3 |
+
# ββ System deps ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 4 |
+
RUN apt-get update && apt-get install -y --no-install-recommends \
|
| 5 |
+
make g++ python3 curl wget ca-certificates \
|
| 6 |
+
&& rm -rf /var/lib/apt/lists/*
|
| 7 |
+
|
| 8 |
+
# ββ CPU Performance ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 9 |
+
ENV NODE_OPTIONS="--max-old-space-size=6144"
|
| 10 |
+
ENV UV_THREADPOOL_SIZE=16
|
| 11 |
+
|
| 12 |
+
WORKDIR /app
|
| 13 |
+
|
| 14 |
+
# ββ Install deps βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 15 |
+
COPY package.json ./
|
| 16 |
+
RUN npm install --production 2>&1 | tail -5 || \
|
| 17 |
+
npm install --legacy-peer-deps 2>&1 | tail -5 || true
|
| 18 |
+
|
| 19 |
+
# ββ Copy app βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 20 |
+
COPY . .
|
| 21 |
+
|
| 22 |
+
# ββ Model cache dir ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 23 |
+
RUN mkdir -p /app/models /tmp/worker-cache
|
| 24 |
+
|
| 25 |
+
ENV PORT=7860
|
| 26 |
+
ENV NODE_ENV=production
|
| 27 |
+
ENV MODEL_DIR=/app/models
|
| 28 |
+
|
| 29 |
+
EXPOSE 7860
|
| 30 |
+
|
| 31 |
+
HEALTHCHECK --interval=30s --timeout=10s --start-period=180s --retries=5 \
|
| 32 |
+
CMD curl -f http://localhost:7860/health || exit 1
|
| 33 |
+
|
| 34 |
+
CMD ["node", "server.js"]
|
package.json
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"name": "rigochat-worker",
|
| 3 |
+
"version": "1.0.0",
|
| 4 |
+
"description": "RigoChat-7B distributed inference worker",
|
| 5 |
+
"type": "module",
|
| 6 |
+
"main": "server.js",
|
| 7 |
+
"dependencies": {
|
| 8 |
+
"node-llama-cpp": "^3.4.0"
|
| 9 |
+
}
|
| 10 |
+
}
|
server.js
ADDED
|
@@ -0,0 +1,232 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
/**
|
| 2 |
+
* RigoChat Worker β Distributed Inference Node
|
| 3 |
+
* ================================================
|
| 4 |
+
* Lightweight HF Space that loads RigoChat-7B-v2 and exposes
|
| 5 |
+
* an /inference endpoint. Multiple workers can be combined
|
| 6 |
+
* via Promise.any for faster responses.
|
| 7 |
+
*/
|
| 8 |
+
|
| 9 |
+
import http from 'http';
|
| 10 |
+
import { existsSync, mkdirSync, readdirSync } from 'fs';
|
| 11 |
+
import path from 'path';
|
| 12 |
+
|
| 13 |
+
// ββ Config ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 14 |
+
const PORT = parseInt(process.env.PORT ?? '7860');
|
| 15 |
+
const MODEL_DIR = process.env.MODEL_DIR ?? './models';
|
| 16 |
+
const MODEL_REPO = process.env.MODEL_REPO ?? 'IIC/RigoChat-7b-v2-GGUF';
|
| 17 |
+
const MODEL_FILE = process.env.MODEL_FILE ?? 'rigochat-7b-v2-Q4_K_M.gguf';
|
| 18 |
+
const CTX_SIZE = parseInt(process.env.CTX_SIZE ?? '4096');
|
| 19 |
+
const WORKER_ID = process.env.SPACE_ID ?? `worker-${Math.random().toString(36).slice(2, 6)}`;
|
| 20 |
+
const AUTH_KEY = process.env.WORKER_KEY ?? 'zelin-cluster'; // Simple auth
|
| 21 |
+
|
| 22 |
+
// ββ State βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 23 |
+
let _model = null;
|
| 24 |
+
let _ready = false;
|
| 25 |
+
let _loading = false;
|
| 26 |
+
let _stats = { requests: 0, totalMs: 0, errors: 0, avgMs: 0 };
|
| 27 |
+
|
| 28 |
+
// ββ Load Model ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 29 |
+
async function loadModel() {
|
| 30 |
+
if (_ready || _loading) return;
|
| 31 |
+
_loading = true;
|
| 32 |
+
const start = Date.now();
|
| 33 |
+
|
| 34 |
+
try {
|
| 35 |
+
console.log(`[Worker] Loading ${MODEL_FILE}...`);
|
| 36 |
+
mkdirSync(MODEL_DIR, { recursive: true });
|
| 37 |
+
|
| 38 |
+
const { getLlama } = await import('node-llama-cpp');
|
| 39 |
+
const llama = await getLlama();
|
| 40 |
+
|
| 41 |
+
const modelPath = path.join(MODEL_DIR, MODEL_FILE);
|
| 42 |
+
|
| 43 |
+
// Download if needed
|
| 44 |
+
if (!existsSync(modelPath)) {
|
| 45 |
+
// Check for HF-prefixed names
|
| 46 |
+
const files = readdirSync(MODEL_DIR).filter(f => f.endsWith('.gguf'));
|
| 47 |
+
const found = files.find(f => f.includes('rigochat') || f.includes('RigoChat'));
|
| 48 |
+
|
| 49 |
+
if (!found) {
|
| 50 |
+
console.log(`[Worker] Downloading ${MODEL_FILE} (~1GB)...`);
|
| 51 |
+
const { createModelDownloader } = await import('node-llama-cpp');
|
| 52 |
+
const dl = await createModelDownloader({
|
| 53 |
+
modelUri: `hf:${MODEL_REPO}/${MODEL_FILE}`,
|
| 54 |
+
dirPath: MODEL_DIR,
|
| 55 |
+
onProgress: ({ downloadedSize, totalSize }) => {
|
| 56 |
+
const pct = totalSize ? Math.round(downloadedSize / totalSize * 100) : '?';
|
| 57 |
+
process.stdout.write(`\r[Worker] Downloading... ${pct}%`);
|
| 58 |
+
},
|
| 59 |
+
});
|
| 60 |
+
await dl.download();
|
| 61 |
+
console.log('\n[Worker] Download complete β
');
|
| 62 |
+
}
|
| 63 |
+
}
|
| 64 |
+
|
| 65 |
+
// Find actual model file
|
| 66 |
+
let actualPath = modelPath;
|
| 67 |
+
if (!existsSync(modelPath)) {
|
| 68 |
+
const files = readdirSync(MODEL_DIR).filter(f => f.endsWith('.gguf'));
|
| 69 |
+
const found = files.find(f => f.includes('rigochat') || f.includes('RigoChat'));
|
| 70 |
+
if (found) actualPath = path.join(MODEL_DIR, found);
|
| 71 |
+
}
|
| 72 |
+
|
| 73 |
+
console.log('[Worker] Loading model into memory...');
|
| 74 |
+
_model = await llama.loadModel({ modelPath: actualPath, gpuLayers: 0 });
|
| 75 |
+
_ready = true;
|
| 76 |
+
_loading = false;
|
| 77 |
+
|
| 78 |
+
const elapsed = ((Date.now() - start) / 1000).toFixed(1);
|
| 79 |
+
console.log(`[Worker] β
Ready in ${elapsed}s β ${WORKER_ID}`);
|
| 80 |
+
} catch (err) {
|
| 81 |
+
_loading = false;
|
| 82 |
+
console.error('[Worker] Load error:', err.message);
|
| 83 |
+
}
|
| 84 |
+
}
|
| 85 |
+
|
| 86 |
+
// ββ Inference βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 87 |
+
async function infer(messages, maxTokens = 300, temperature = 0.7) {
|
| 88 |
+
if (!_model) throw new Error('Model not loaded');
|
| 89 |
+
|
| 90 |
+
const { LlamaChatSession } = await import('node-llama-cpp');
|
| 91 |
+
const ctx = await _model.createContext({ contextSize: CTX_SIZE });
|
| 92 |
+
|
| 93 |
+
const systemMsg = messages.find(m => m.role === 'system')?.content ?? '';
|
| 94 |
+
const stylePrefix = 'Responde en espaΓ±ol casual argentino. MΓ‘x 2 lΓneas. Sin mayΓΊsculas al inicio. Sin punto final.\n\n';
|
| 95 |
+
const sysFinal = systemMsg.includes('espaΓ±ol') ? systemMsg : stylePrefix + systemMsg;
|
| 96 |
+
|
| 97 |
+
const session = new LlamaChatSession({
|
| 98 |
+
contextSequence: ctx.getSequence(),
|
| 99 |
+
systemPrompt: sysFinal,
|
| 100 |
+
});
|
| 101 |
+
|
| 102 |
+
const userMsgs = messages.filter(m => m.role !== 'system');
|
| 103 |
+
const lastUser = userMsgs[userMsgs.length - 1];
|
| 104 |
+
|
| 105 |
+
// Feed history
|
| 106 |
+
for (const msg of userMsgs.slice(0, -1)) {
|
| 107 |
+
await session.prompt(msg.content ?? '', { maxTokens: 1 }).catch(() => {});
|
| 108 |
+
}
|
| 109 |
+
|
| 110 |
+
let result = '';
|
| 111 |
+
if (lastUser) {
|
| 112 |
+
result = await session.prompt(lastUser.content ?? '', {
|
| 113 |
+
maxTokens,
|
| 114 |
+
temperature,
|
| 115 |
+
topP: 0.9,
|
| 116 |
+
topK: 40,
|
| 117 |
+
minP: 0.05,
|
| 118 |
+
repeatPenalty: { penalty: 1.35, lastTokens: 96, frequencyPenalty: 0.1, presencePenalty: 0.05 },
|
| 119 |
+
});
|
| 120 |
+
}
|
| 121 |
+
|
| 122 |
+
session.dispose?.();
|
| 123 |
+
ctx.dispose?.();
|
| 124 |
+
|
| 125 |
+
return result.trim();
|
| 126 |
+
}
|
| 127 |
+
|
| 128 |
+
// ββ HTTP Server βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 129 |
+
const server = http.createServer(async (req, res) => {
|
| 130 |
+
// CORS
|
| 131 |
+
res.setHeader('Access-Control-Allow-Origin', '*');
|
| 132 |
+
res.setHeader('Access-Control-Allow-Methods', 'GET, POST, OPTIONS');
|
| 133 |
+
res.setHeader('Access-Control-Allow-Headers', 'Content-Type, Authorization');
|
| 134 |
+
|
| 135 |
+
if (req.method === 'OPTIONS') { res.writeHead(204); res.end(); return; }
|
| 136 |
+
|
| 137 |
+
const url = new URL(req.url, `http://localhost:${PORT}`);
|
| 138 |
+
|
| 139 |
+
// Health check
|
| 140 |
+
if (url.pathname === '/health') {
|
| 141 |
+
res.writeHead(200, { 'Content-Type': 'application/json' });
|
| 142 |
+
res.end(JSON.stringify({
|
| 143 |
+
status: _ready ? 'ready' : (_loading ? 'loading' : 'error'),
|
| 144 |
+
worker: WORKER_ID,
|
| 145 |
+
model: MODEL_FILE,
|
| 146 |
+
uptime: process.uptime(),
|
| 147 |
+
memory: Math.round(process.memoryUsage().heapUsed / 1024 / 1024) + 'MB',
|
| 148 |
+
stats: _stats,
|
| 149 |
+
}));
|
| 150 |
+
return;
|
| 151 |
+
}
|
| 152 |
+
|
| 153 |
+
// Inference endpoint
|
| 154 |
+
if (url.pathname === '/inference' && req.method === 'POST') {
|
| 155 |
+
// Simple auth
|
| 156 |
+
const auth = req.headers['authorization'];
|
| 157 |
+
if (auth !== `Bearer ${AUTH_KEY}`) {
|
| 158 |
+
res.writeHead(401, { 'Content-Type': 'application/json' });
|
| 159 |
+
res.end(JSON.stringify({ error: 'Unauthorized' }));
|
| 160 |
+
return;
|
| 161 |
+
}
|
| 162 |
+
|
| 163 |
+
if (!_ready) {
|
| 164 |
+
res.writeHead(503, { 'Content-Type': 'application/json' });
|
| 165 |
+
res.end(JSON.stringify({ error: 'Model not ready', status: _loading ? 'loading' : 'error' }));
|
| 166 |
+
return;
|
| 167 |
+
}
|
| 168 |
+
|
| 169 |
+
try {
|
| 170 |
+
const body = await new Promise((resolve, reject) => {
|
| 171 |
+
let data = '';
|
| 172 |
+
req.on('data', c => data += c);
|
| 173 |
+
req.on('end', () => resolve(JSON.parse(data)));
|
| 174 |
+
req.on('error', reject);
|
| 175 |
+
setTimeout(() => reject(new Error('Timeout')), 30000);
|
| 176 |
+
});
|
| 177 |
+
|
| 178 |
+
const { messages, maxTokens = 300, temperature = 0.7 } = body;
|
| 179 |
+
if (!messages?.length) {
|
| 180 |
+
res.writeHead(400, { 'Content-Type': 'application/json' });
|
| 181 |
+
res.end(JSON.stringify({ error: 'messages required' }));
|
| 182 |
+
return;
|
| 183 |
+
}
|
| 184 |
+
|
| 185 |
+
const start = Date.now();
|
| 186 |
+
const result = await infer(messages, maxTokens, temperature);
|
| 187 |
+
const ms = Date.now() - start;
|
| 188 |
+
|
| 189 |
+
_stats.requests++;
|
| 190 |
+
_stats.totalMs += ms;
|
| 191 |
+
_stats.avgMs = Math.round(_stats.totalMs / _stats.requests);
|
| 192 |
+
|
| 193 |
+
res.writeHead(200, { 'Content-Type': 'application/json' });
|
| 194 |
+
res.end(JSON.stringify({
|
| 195 |
+
result,
|
| 196 |
+
worker: WORKER_ID,
|
| 197 |
+
latencyMs: ms,
|
| 198 |
+
tokens: result.split(/\s+/).length,
|
| 199 |
+
}));
|
| 200 |
+
} catch (err) {
|
| 201 |
+
_stats.errors++;
|
| 202 |
+
res.writeHead(500, { 'Content-Type': 'application/json' });
|
| 203 |
+
res.end(JSON.stringify({ error: err.message, worker: WORKER_ID }));
|
| 204 |
+
}
|
| 205 |
+
return;
|
| 206 |
+
}
|
| 207 |
+
|
| 208 |
+
// Root
|
| 209 |
+
if (url.pathname === '/') {
|
| 210 |
+
res.writeHead(200, { 'Content-Type': 'application/json' });
|
| 211 |
+
res.end(JSON.stringify({
|
| 212 |
+
name: 'rigochat-worker',
|
| 213 |
+
version: '1.0.0',
|
| 214 |
+
status: _ready ? 'ready' : (_loading ? 'loading' : 'error'),
|
| 215 |
+
worker: WORKER_ID,
|
| 216 |
+
model: MODEL_FILE,
|
| 217 |
+
endpoints: ['/health', '/inference'],
|
| 218 |
+
}));
|
| 219 |
+
return;
|
| 220 |
+
}
|
| 221 |
+
|
| 222 |
+
res.writeHead(404);
|
| 223 |
+
res.end('Not found');
|
| 224 |
+
});
|
| 225 |
+
|
| 226 |
+
// ββ Start βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 227 |
+
server.listen(PORT, '0.0.0.0', () => {
|
| 228 |
+
console.log(`[Worker] HTTP server on port ${PORT}`);
|
| 229 |
+
console.log(`[Worker] Worker ID: ${WORKER_ID}`);
|
| 230 |
+
console.log(`[Worker] Loading model in background...`);
|
| 231 |
+
loadModel();
|
| 232 |
+
});
|