Download server.js from TomatitoToho/rigochat-node-4: direct link, hf CLI and curl.
- Browser
- Download file 8.85 kB
-
https://huggingface.co/spaces/TomatitoToho/rigochat-node-4/resolve/main/server.js
- Command line
-
hf download hf://spaces/TomatitoToho/rigochat-node-4/server.js
-
curl -L -o server.js https://huggingface.co/spaces/TomatitoToho/rigochat-node-4/resolve/main/server.js
8.85 kB
| /** | |
| * RigoChat Worker β Distributed Inference Node | |
| * ================================================ | |
| * Lightweight HF Space that loads RigoChat-7B-v2 and exposes | |
| * an /inference endpoint. Multiple workers can be combined | |
| * via Promise.any for faster responses. | |
| */ | |
| import http from 'http'; | |
| import { existsSync, mkdirSync, readdirSync } from 'fs'; | |
| import path from 'path'; | |
| // ββ Config ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| const PORT = parseInt(process.env.PORT ?? '7860'); | |
| const MODEL_DIR = process.env.MODEL_DIR ?? './models'; | |
| const MODEL_REPO = process.env.MODEL_REPO ?? 'IIC/RigoChat-7b-v2-GGUF'; | |
| const MODEL_FILE = process.env.MODEL_FILE ?? 'rigochat-7b-v2-Q4_K_M.gguf'; | |
| const CTX_SIZE = parseInt(process.env.CTX_SIZE ?? '4096'); | |
| const WORKER_ID = process.env.SPACE_ID ?? `worker-${Math.random().toString(36).slice(2, 6)}`; | |
| const AUTH_KEY = process.env.WORKER_KEY ?? 'zelin-cluster'; // Simple auth | |
| // ββ State βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| let _model = null; | |
| let _ready = false; | |
| let _loading = false; | |
| let _stats = { requests: 0, totalMs: 0, errors: 0, avgMs: 0 }; | |
| // ββ Load Model ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| async function loadModel() { | |
| if (_ready || _loading) return; | |
| _loading = true; | |
| const start = Date.now(); | |
| try { | |
| console.log(`[Worker] Loading ${MODEL_FILE}...`); | |
| mkdirSync(MODEL_DIR, { recursive: true }); | |
| const { getLlama } = await import('node-llama-cpp'); | |
| const llama = await getLlama(); | |
| const modelPath = path.join(MODEL_DIR, MODEL_FILE); | |
| // Download if needed | |
| if (!existsSync(modelPath)) { | |
| // Check for HF-prefixed names | |
| const files = readdirSync(MODEL_DIR).filter(f => f.endsWith('.gguf')); | |
| const found = files.find(f => f.includes('rigochat') || f.includes('RigoChat')); | |
| if (!found) { | |
| console.log(`[Worker] Downloading ${MODEL_FILE} (~1GB)...`); | |
| const { createModelDownloader } = await import('node-llama-cpp'); | |
| const dl = await createModelDownloader({ | |
| modelUri: `hf:${MODEL_REPO}/${MODEL_FILE}`, | |
| dirPath: MODEL_DIR, | |
| onProgress: ({ downloadedSize, totalSize }) => { | |
| const pct = totalSize ? Math.round(downloadedSize / totalSize * 100) : '?'; | |
| process.stdout.write(`\r[Worker] Downloading... ${pct}%`); | |
| }, | |
| }); | |
| await dl.download(); | |
| console.log('\n[Worker] Download complete β '); | |
| } | |
| } | |
| // Find actual model file | |
| let actualPath = modelPath; | |
| if (!existsSync(modelPath)) { | |
| const files = readdirSync(MODEL_DIR).filter(f => f.endsWith('.gguf')); | |
| const found = files.find(f => f.includes('rigochat') || f.includes('RigoChat')); | |
| if (found) actualPath = path.join(MODEL_DIR, found); | |
| } | |
| console.log('[Worker] Loading model into memory...'); | |
| _model = await llama.loadModel({ modelPath: actualPath, gpuLayers: 0 }); | |
| _ready = true; | |
| _loading = false; | |
| const elapsed = ((Date.now() - start) / 1000).toFixed(1); | |
| console.log(`[Worker] β Ready in ${elapsed}s β ${WORKER_ID}`); | |
| } catch (err) { | |
| _loading = false; | |
| console.error('[Worker] Load error:', err.message); | |
| } | |
| } | |
| // ββ Inference βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| async function infer(messages, maxTokens = 300, temperature = 0.7) { | |
| if (!_model) throw new Error('Model not loaded'); | |
| const { LlamaChatSession } = await import('node-llama-cpp'); | |
| const ctx = await _model.createContext({ contextSize: CTX_SIZE }); | |
| const systemMsg = messages.find(m => m.role === 'system')?.content ?? ''; | |
| const stylePrefix = 'Responde en espaΓ±ol casual argentino. MΓ‘x 2 lΓneas. Sin mayΓΊsculas al inicio. Sin punto final.\n\n'; | |
| const sysFinal = systemMsg.includes('espaΓ±ol') ? systemMsg : stylePrefix + systemMsg; | |
| const session = new LlamaChatSession({ | |
| contextSequence: ctx.getSequence(), | |
| systemPrompt: sysFinal, | |
| }); | |
| const userMsgs = messages.filter(m => m.role !== 'system'); | |
| const lastUser = userMsgs[userMsgs.length - 1]; | |
| // Feed history | |
| for (const msg of userMsgs.slice(0, -1)) { | |
| await session.prompt(msg.content ?? '', { maxTokens: 1 }).catch(() => {}); | |
| } | |
| let result = ''; | |
| if (lastUser) { | |
| result = await session.prompt(lastUser.content ?? '', { | |
| maxTokens, | |
| temperature, | |
| topP: 0.9, | |
| topK: 40, | |
| minP: 0.05, | |
| repeatPenalty: { penalty: 1.35, lastTokens: 96, frequencyPenalty: 0.1, presencePenalty: 0.05 }, | |
| }); | |
| } | |
| session.dispose?.(); | |
| ctx.dispose?.(); | |
| return result.trim(); | |
| } | |
| // ββ HTTP Server βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| const server = http.createServer(async (req, res) => { | |
| // CORS | |
| res.setHeader('Access-Control-Allow-Origin', '*'); | |
| res.setHeader('Access-Control-Allow-Methods', 'GET, POST, OPTIONS'); | |
| res.setHeader('Access-Control-Allow-Headers', 'Content-Type, Authorization'); | |
| if (req.method === 'OPTIONS') { res.writeHead(204); res.end(); return; } | |
| const url = new URL(req.url, `http://localhost:${PORT}`); | |
| // Health check | |
| if (url.pathname === '/health') { | |
| res.writeHead(200, { 'Content-Type': 'application/json' }); | |
| res.end(JSON.stringify({ | |
| status: _ready ? 'ready' : (_loading ? 'loading' : 'error'), | |
| worker: WORKER_ID, | |
| model: MODEL_FILE, | |
| uptime: process.uptime(), | |
| memory: Math.round(process.memoryUsage().heapUsed / 1024 / 1024) + 'MB', | |
| stats: _stats, | |
| })); | |
| return; | |
| } | |
| // Inference endpoint | |
| if (url.pathname === '/inference' && req.method === 'POST') { | |
| // Simple auth | |
| const auth = req.headers['authorization']; | |
| if (auth !== `Bearer ${AUTH_KEY}`) { | |
| res.writeHead(401, { 'Content-Type': 'application/json' }); | |
| res.end(JSON.stringify({ error: 'Unauthorized' })); | |
| return; | |
| } | |
| if (!_ready) { | |
| res.writeHead(503, { 'Content-Type': 'application/json' }); | |
| res.end(JSON.stringify({ error: 'Model not ready', status: _loading ? 'loading' : 'error' })); | |
| return; | |
| } | |
| try { | |
| const body = await new Promise((resolve, reject) => { | |
| let data = ''; | |
| req.on('data', c => data += c); | |
| req.on('end', () => resolve(JSON.parse(data))); | |
| req.on('error', reject); | |
| setTimeout(() => reject(new Error('Timeout')), 30000); | |
| }); | |
| const { messages, maxTokens = 300, temperature = 0.7 } = body; | |
| if (!messages?.length) { | |
| res.writeHead(400, { 'Content-Type': 'application/json' }); | |
| res.end(JSON.stringify({ error: 'messages required' })); | |
| return; | |
| } | |
| const start = Date.now(); | |
| const result = await infer(messages, maxTokens, temperature); | |
| const ms = Date.now() - start; | |
| _stats.requests++; | |
| _stats.totalMs += ms; | |
| _stats.avgMs = Math.round(_stats.totalMs / _stats.requests); | |
| res.writeHead(200, { 'Content-Type': 'application/json' }); | |
| res.end(JSON.stringify({ | |
| result, | |
| worker: WORKER_ID, | |
| latencyMs: ms, | |
| tokens: result.split(/\s+/).length, | |
| })); | |
| } catch (err) { | |
| _stats.errors++; | |
| res.writeHead(500, { 'Content-Type': 'application/json' }); | |
| res.end(JSON.stringify({ error: err.message, worker: WORKER_ID })); | |
| } | |
| return; | |
| } | |
| // Root | |
| if (url.pathname === '/') { | |
| res.writeHead(200, { 'Content-Type': 'application/json' }); | |
| res.end(JSON.stringify({ | |
| name: 'rigochat-worker', | |
| version: '1.0.0', | |
| status: _ready ? 'ready' : (_loading ? 'loading' : 'error'), | |
| worker: WORKER_ID, | |
| model: MODEL_FILE, | |
| endpoints: ['/health', '/inference'], | |
| })); | |
| return; | |
| } | |
| res.writeHead(404); | |
| res.end('Not found'); | |
| }); | |
| // ββ Start βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| server.listen(PORT, '0.0.0.0', () => { | |
| console.log(`[Worker] HTTP server on port ${PORT}`); | |
| console.log(`[Worker] Worker ID: ${WORKER_ID}`); | |
| console.log(`[Worker] Loading model in background...`); | |
| loadModel(); | |
| }); | |