/** * Smart Text Preprocessor & Intent Router (Claude-Style Pipeline) * * NO hardcoded word replacement dictionaries. * * Features: * 1. Safe Unicode sanitization (stripping zero-width spaces, invisible control characters). * 2. Intent Routing (the "IF-uri" inspired by Claude's harness: Coding, Factual, Reasoning, Chat). * 3. Language Detection (Romanian vs English contextual detection). * 4. Neural Coprocessor Hook: Supports delegating prompt refinement and query rewriting * to a secondary neural model (e.g., local GEC/SLM sidecar or fast LLM endpoint). */ export interface PreprocessedInput { rawInput: string; cleanedText: string; searchQuery: string; intent: "greeting" | "code" | "reasoning" | "factual_search" | "creative" | "general"; language: "ro" | "en"; suggestedStartMsgPrompt?: string; } /** * Normalizes unicode, invisible characters, and whitespace safely * without altering user words or colloquial phrasing. */ export function sanitizeText(text: string): string { if (!text) return ""; return text // Strip zero-width spaces, invisible characters and directional marks .replace(/[\u0000-\u0008\u000B\u000C\u000E-\u001F\u007F-\u009F\u200B-\u200D\uFEFF]/g, "") // Normalize abnormal unicode space characters to standard space .replace(/[\u00A0\u1680\u180E\u2000-\u200A\u202F\u205F\u3000]/g, " ") // Normalize curly quotes into clean ASCII quotes .replace(/[“”„‟]/g, '"') .replace(/[‘’‚‛`]/g, "'") // Collapse excessive whitespace .replace(/[ \t]+/g, " ") .replace(/\n{3,}/g, "\n\n") .trim(); } /** * Contextual language detection (Romanian vs English) based on function words. */ export function detectLanguage(text: string): "ro" | "en" { const lower = text.toLowerCase(); const roMarkers = /\b(cum|ce|de|la|pe|in|și|si|sau|este|sunt|nu|mai|care|pentru|despre|vreau|pot|face|bine|mersi|salut|bună|buna|mulțumesc|multumesc)\b/i; const enMarkers = /\b(the|is|are|and|or|not|can|what|how|why|who|where|when|this|that|with|from|have|has|would|could)\b/i; const roMatches = (lower.match(roMarkers) || []).length; const enMatches = (lower.match(enMarkers) || []).length; if (roMatches > enMatches) return "ro"; if (enMatches > roMatches) return "en"; if (/[ăâîșț]/i.test(text)) return "ro"; return "en"; } /** * Extracts a focused search query for RAG and Web Search * by stripping conversational filler prefixes if present. */ export function extractCleanSearchQuery(text: string): string { let q = sanitizeText(text); // Strip conversational intro wrappers without altering the core topic const prefixRegex = /^(hei|salut|buna|te rog|poti sa|poti|cauta|cauta pe net|cauta pe google|spune-mi despre|can you tell me about|search for|look up|tell me about)\s+/i; q = q.replace(prefixRegex, ""); q = q.replace(/[?!.]+$/, "").trim(); return q.length > 2 ? q : sanitizeText(text); } /** * Claude-Style Intent Routing ("IF-urile"): * Analyzes the user's intent to dynamically select the best framing and tool activation. */ export function classifyIntent(text: string, language: "ro" | "en"): { intent: PreprocessedInput["intent"]; suggestedStartMsgPrompt?: string; } { const lower = text.toLowerCase(); // IF 1: Greeting / Short conversational chitchat const isGreeting = /^(salut|buna|bună|hei|hey|hello|hi|cf|ce faci|ce mai faci|buna ziua|bună ziua)[.?! ]*$/i.test(lower); if (isGreeting) { return { intent: "greeting", suggestedStartMsgPrompt: "You are a friendly, concise assistant. Greet warmly and keep the answer brief.", }; } // IF 2: Coding, Software Engineering, Bugs & Syntax if ( /\b(python|javascript|typescript|c\+\+|rust|html|css|sql|function|def |class |error|bug|code|script|api|algoritm|compil|react|next\.?js)\b/i.test(lower) ) { return { intent: "code", suggestedStartMsgPrompt: "You are an expert software engineer. Provide direct, clean code and concise explanations without fluff.", }; } // IF 3: Step-by-step Reasoning / Logic / Math if ( /\b(de ce|cum se explica|calculeaza|demonstreaza|logica|pas cu pas|rezolva|why|calculate|solve|step by step|proof|math|reasoning)\b/i.test(lower) || /[0-9]+\s*[\+\-\*\/]\s*[0-9]+/.test(lower) ) { return { intent: "reasoning", suggestedStartMsgPrompt: "Think step-by-step with structured logical reasoning before providing the final concise answer.", }; } // IF 4: Real-time Facts / News / Weather / Prices / Current Events if ( /\b(stiri|știri|cine este|cine a fost|ce vreme|pretul|cât costă|azi|recent|data de azi|news|weather|price|who is|latest|current|today)\b/i.test(lower) ) { return { intent: "factual_search", suggestedStartMsgPrompt: "You are a factual assistant. Base your answers strictly on verified evidence and state facts concisely.", }; } // IF 5: Storytelling / Creativity if ( /\b(poveste|poezie|creeaza o poveste|imagineaza|gluma|story|poem|write a story|creative|joke)\b/i.test(lower) ) { return { intent: "creative", suggestedStartMsgPrompt: "Be engaging, imaginative, and creative while crafting a delightful narrative.", }; } return { intent: "general" }; } /** * Main Preprocessing Pipeline: Runs sanitization, language detection, * query extraction and intent routing without any hardcoded dictionary swaps. */ export function preprocessUserInput(rawInput: string): PreprocessedInput { const cleanedText = sanitizeText(rawInput); const language = detectLanguage(cleanedText); const searchQuery = extractCleanSearchQuery(cleanedText); const { intent, suggestedStartMsgPrompt } = classifyIntent(cleanedText, language); return { rawInput, cleanedText, searchQuery, intent, language, suggestedStartMsgPrompt, }; } /** * Asynchronous Smart Neural Preprocessor: * Calls the local GPU sidecar model (/v1/refine) powered by Qwen on GPU * to perform zero-hardcoded grammatical error correction and intent extraction. * Falls back gracefully to heuristic sanitization if server is unavailable or slow. */ export async function refinePromptWithNeuralEngine(rawInput: string): Promise { const base = preprocessUserInput(rawInput); // Romanian language protection: small English-centric SLMs (like Qwen 0.5B) // tend to hallucinate Romanian words (e.g. 'pamantul' -> 'pancake'). // For Romanian, preserve authentic user input and use accurate semantic intent classification. if (base.language === "ro") { return base; } const PY_SERVER = process.env.PURCAR_INFERENCE_API_URL || "http://localhost:8000"; try { const ctrl = new AbortController(); const timer = setTimeout(() => ctrl.abort(), 1500); const res = await fetch(`${PY_SERVER}/v1/refine`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ prompt: rawInput }), signal: ctrl.signal, }); clearTimeout(timer); if (res.ok) { const data = await res.json(); if (data.refined_prompt && data.refined_prompt.trim()) { const refinedClean = sanitizeText(data.refined_prompt); const lang = detectLanguage(refinedClean); const { suggestedStartMsgPrompt } = classifyIntent(refinedClean, lang); return { rawInput, cleanedText: refinedClean, searchQuery: data.search_query ? sanitizeText(data.search_query) : base.searchQuery, intent: (data.intent as any) || base.intent, language: lang, suggestedStartMsgPrompt: suggestedStartMsgPrompt || base.suggestedStartMsgPrompt, }; } } } catch { // Graceful fallback to deterministic preprocessor } return base; }