File size: 8,851 Bytes
d59cbfd | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 | /**
* RigoChat Worker β Distributed Inference Node
* ================================================
* Lightweight HF Space that loads RigoChat-7B-v2 and exposes
* an /inference endpoint. Multiple workers can be combined
* via Promise.any for faster responses.
*/
import http from 'http';
import { existsSync, mkdirSync, readdirSync } from 'fs';
import path from 'path';
// ββ Config ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
const PORT = parseInt(process.env.PORT ?? '7860');
const MODEL_DIR = process.env.MODEL_DIR ?? './models';
const MODEL_REPO = process.env.MODEL_REPO ?? 'IIC/RigoChat-7b-v2-GGUF';
const MODEL_FILE = process.env.MODEL_FILE ?? 'rigochat-7b-v2-Q4_K_M.gguf';
const CTX_SIZE = parseInt(process.env.CTX_SIZE ?? '4096');
const WORKER_ID = process.env.SPACE_ID ?? `worker-${Math.random().toString(36).slice(2, 6)}`;
const AUTH_KEY = process.env.WORKER_KEY ?? 'zelin-cluster'; // Simple auth
// ββ State βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
let _model = null;
let _ready = false;
let _loading = false;
let _stats = { requests: 0, totalMs: 0, errors: 0, avgMs: 0 };
// ββ Load Model ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
async function loadModel() {
if (_ready || _loading) return;
_loading = true;
const start = Date.now();
try {
console.log(`[Worker] Loading ${MODEL_FILE}...`);
mkdirSync(MODEL_DIR, { recursive: true });
const { getLlama } = await import('node-llama-cpp');
const llama = await getLlama();
const modelPath = path.join(MODEL_DIR, MODEL_FILE);
// Download if needed
if (!existsSync(modelPath)) {
// Check for HF-prefixed names
const files = readdirSync(MODEL_DIR).filter(f => f.endsWith('.gguf'));
const found = files.find(f => f.includes('rigochat') || f.includes('RigoChat'));
if (!found) {
console.log(`[Worker] Downloading ${MODEL_FILE} (~1GB)...`);
const { createModelDownloader } = await import('node-llama-cpp');
const dl = await createModelDownloader({
modelUri: `hf:${MODEL_REPO}/${MODEL_FILE}`,
dirPath: MODEL_DIR,
onProgress: ({ downloadedSize, totalSize }) => {
const pct = totalSize ? Math.round(downloadedSize / totalSize * 100) : '?';
process.stdout.write(`\r[Worker] Downloading... ${pct}%`);
},
});
await dl.download();
console.log('\n[Worker] Download complete β
');
}
}
// Find actual model file
let actualPath = modelPath;
if (!existsSync(modelPath)) {
const files = readdirSync(MODEL_DIR).filter(f => f.endsWith('.gguf'));
const found = files.find(f => f.includes('rigochat') || f.includes('RigoChat'));
if (found) actualPath = path.join(MODEL_DIR, found);
}
console.log('[Worker] Loading model into memory...');
_model = await llama.loadModel({ modelPath: actualPath, gpuLayers: 0 });
_ready = true;
_loading = false;
const elapsed = ((Date.now() - start) / 1000).toFixed(1);
console.log(`[Worker] β
Ready in ${elapsed}s β ${WORKER_ID}`);
} catch (err) {
_loading = false;
console.error('[Worker] Load error:', err.message);
}
}
// ββ Inference βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
async function infer(messages, maxTokens = 300, temperature = 0.7) {
if (!_model) throw new Error('Model not loaded');
const { LlamaChatSession } = await import('node-llama-cpp');
const ctx = await _model.createContext({ contextSize: CTX_SIZE });
const systemMsg = messages.find(m => m.role === 'system')?.content ?? '';
const stylePrefix = 'Responde en espaΓ±ol casual argentino. MΓ‘x 2 lΓneas. Sin mayΓΊsculas al inicio. Sin punto final.\n\n';
const sysFinal = systemMsg.includes('espaΓ±ol') ? systemMsg : stylePrefix + systemMsg;
const session = new LlamaChatSession({
contextSequence: ctx.getSequence(),
systemPrompt: sysFinal,
});
const userMsgs = messages.filter(m => m.role !== 'system');
const lastUser = userMsgs[userMsgs.length - 1];
// Feed history
for (const msg of userMsgs.slice(0, -1)) {
await session.prompt(msg.content ?? '', { maxTokens: 1 }).catch(() => {});
}
let result = '';
if (lastUser) {
result = await session.prompt(lastUser.content ?? '', {
maxTokens,
temperature,
topP: 0.9,
topK: 40,
minP: 0.05,
repeatPenalty: { penalty: 1.35, lastTokens: 96, frequencyPenalty: 0.1, presencePenalty: 0.05 },
});
}
session.dispose?.();
ctx.dispose?.();
return result.trim();
}
// ββ HTTP Server βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
const server = http.createServer(async (req, res) => {
// CORS
res.setHeader('Access-Control-Allow-Origin', '*');
res.setHeader('Access-Control-Allow-Methods', 'GET, POST, OPTIONS');
res.setHeader('Access-Control-Allow-Headers', 'Content-Type, Authorization');
if (req.method === 'OPTIONS') { res.writeHead(204); res.end(); return; }
const url = new URL(req.url, `http://localhost:${PORT}`);
// Health check
if (url.pathname === '/health') {
res.writeHead(200, { 'Content-Type': 'application/json' });
res.end(JSON.stringify({
status: _ready ? 'ready' : (_loading ? 'loading' : 'error'),
worker: WORKER_ID,
model: MODEL_FILE,
uptime: process.uptime(),
memory: Math.round(process.memoryUsage().heapUsed / 1024 / 1024) + 'MB',
stats: _stats,
}));
return;
}
// Inference endpoint
if (url.pathname === '/inference' && req.method === 'POST') {
// Simple auth
const auth = req.headers['authorization'];
if (auth !== `Bearer ${AUTH_KEY}`) {
res.writeHead(401, { 'Content-Type': 'application/json' });
res.end(JSON.stringify({ error: 'Unauthorized' }));
return;
}
if (!_ready) {
res.writeHead(503, { 'Content-Type': 'application/json' });
res.end(JSON.stringify({ error: 'Model not ready', status: _loading ? 'loading' : 'error' }));
return;
}
try {
const body = await new Promise((resolve, reject) => {
let data = '';
req.on('data', c => data += c);
req.on('end', () => resolve(JSON.parse(data)));
req.on('error', reject);
setTimeout(() => reject(new Error('Timeout')), 30000);
});
const { messages, maxTokens = 300, temperature = 0.7 } = body;
if (!messages?.length) {
res.writeHead(400, { 'Content-Type': 'application/json' });
res.end(JSON.stringify({ error: 'messages required' }));
return;
}
const start = Date.now();
const result = await infer(messages, maxTokens, temperature);
const ms = Date.now() - start;
_stats.requests++;
_stats.totalMs += ms;
_stats.avgMs = Math.round(_stats.totalMs / _stats.requests);
res.writeHead(200, { 'Content-Type': 'application/json' });
res.end(JSON.stringify({
result,
worker: WORKER_ID,
latencyMs: ms,
tokens: result.split(/\s+/).length,
}));
} catch (err) {
_stats.errors++;
res.writeHead(500, { 'Content-Type': 'application/json' });
res.end(JSON.stringify({ error: err.message, worker: WORKER_ID }));
}
return;
}
// Root
if (url.pathname === '/') {
res.writeHead(200, { 'Content-Type': 'application/json' });
res.end(JSON.stringify({
name: 'rigochat-worker',
version: '1.0.0',
status: _ready ? 'ready' : (_loading ? 'loading' : 'error'),
worker: WORKER_ID,
model: MODEL_FILE,
endpoints: ['/health', '/inference'],
}));
return;
}
res.writeHead(404);
res.end('Not found');
});
// ββ Start βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
server.listen(PORT, '0.0.0.0', () => {
console.log(`[Worker] HTTP server on port ${PORT}`);
console.log(`[Worker] Worker ID: ${WORKER_ID}`);
console.log(`[Worker] Loading model in background...`);
loadModel();
});
|