const API_URL = process.env.INFERENCE_API_URL; const API_KEY = process.env.INFERENCE_API_KEY; const FETCH_TIMEOUT = 30000; let currentModel = process.env.INFERENCE_MODEL || 'llama-3.1-8b-instant'; const AVAILABLE_MODELS = [ 'lightning', 'gpt-4o-mini', 'claude-sonnet-4-20250514', 'llama-3.3-70b-versatile', 'llama-3.1-8b-instant', 'gemma2-9b-it', ]; function getModel() { return currentModel; } function setModel(name) { if (name && typeof name === 'string') { currentModel = name.trim(); } } if (!API_URL) { console.warn('[inferenceClient] INFERENCE_API_URL is not set - chat requests will fail.'); } if (!API_KEY) { console.warn('[inferenceClient] INFERENCE_API_KEY is not set - chat requests will fail.'); } async function chat(messages) { const body = { model: currentModel, messages, }; let res; try { const controller = new AbortController(); const timeout = setTimeout(() => controller.abort(), FETCH_TIMEOUT); res = await fetch(API_URL, { method: 'POST', headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${API_KEY}`, }, body: JSON.stringify(body), signal: controller.signal, }); clearTimeout(timeout); } catch (e) { if (e.name === 'AbortError') { throw new Error(`Inference request timed out after ${FETCH_TIMEOUT / 1000}s`); } throw new Error(`Could not reach the Inference Port API: ${e.message}`); } if (!res.ok) { let detail = ''; try { detail = await res.text(); } catch { /* ignore */ } throw new Error(`Inference Port API returned ${res.status}: ${detail.slice(0, 500)}`); } const data = await res.json(); const choice = data.choices && data.choices[0]; if (!choice) { throw new Error('Inference Port API returned no choices.'); } return choice.message; } module.exports = { chat, getModel, setModel, AVAILABLE_MODELS, API_URL };