diff --git "a/ChatIPC.cpp" "b/ChatIPC.cpp"
--- "a/ChatIPC.cpp"
+++ "b/ChatIPC.cpp"
@@ -1,1810 +1,1843 @@
-#include <algorithm>
-#include <cctype>
-#include <cstddef>
-#include <cstdint>
-#include <cstdio>
-#include <fstream>
-#include <iostream>
-#include <iterator>
-#include <mutex>
-#include <optional>
-#include <sstream>
-#include <stdexcept>
-#include <string>
-#include <unordered_map>
-#include <unordered_set>
-#include <utility>
-#include <vector>
-
-#ifdef _OPENMP
-#include <omp.h>
-#else
-inline int omp_get_max_threads(){ return 1; }
-#endif
-
-extern unsigned char dictionary_json[];
-extern unsigned int dictionary_json_len;
-
-
-static inline bool is_space(char c){ return std::isspace(static_cast<unsigned char>(c)) != 0; }
-static inline char to_low(char c){ return static_cast<char>(std::tolower(static_cast<unsigned char>(c))); }
-
-struct DictionaryEntry {
-    std::string pos;
-    std::string word;
-    std::vector<std::string> definitions;
-};
-
-static std::vector<DictionaryEntry> global_dictionary_entries;
-static std::unordered_map<std::string, std::vector<std::string>> global_def_tokens_cache;
-static std::unordered_map<std::string, std::vector<std::string>> global_pos_cache;
-
-static inline bool is_word_char_for_key(char c){
-    unsigned char uc = static_cast<unsigned char>(c);
-    return std::isalnum(uc) != 0 || c == '\'' || c == '-';
-}
-
-static std::string normalize_dictionary_key(const std::string &s){
-    size_t b = 0, e = s.size();
-    while (b < e && !is_word_char_for_key(s[b])) ++b;
-    while (e > b && !is_word_char_for_key(s[e - 1])) --e;
-
-    std::string out;
-    out.reserve(e - b);
-    for (size_t i = b; i < e; ++i) out.push_back(to_low(s[i]));
-    return out;
-}
-
-static std::string normalize_pos_tag(const std::string &s){
-    std::string out;
-    out.reserve(s.size());
-    for (char c : s){
-        unsigned char uc = static_cast<unsigned char>(c);
-        if (std::isalpha(uc) != 0) out.push_back(to_low(c));
-    }
-    return out;
-}
-
-enum class PosClass {
-    Unknown,
-    Noun,
-    Verb,
-    Adj,
-    Adv,
-    Pron,
-    Prep,
-    Conj,
-    Det,
-    Num,
-    Interj
-};
-
-static PosClass pos_class_from_tag(const std::string &tag){
-    if (tag == "n" || tag == "noun") return PosClass::Noun;
-    if (tag == "v" || tag == "verb" || tag == "part" || tag == "participle" || tag == "p") return PosClass::Verb;
-    if (tag == "a" || tag == "adj" || tag == "adjective") return PosClass::Adj;
-    if (tag == "adv" || tag == "adverb") return PosClass::Adv;
-    if (tag == "pron" || tag == "pronoun") return PosClass::Pron;
-    if (tag == "prep" || tag == "preposition") return PosClass::Prep;
-    if (tag == "conj" || tag == "conjunction") return PosClass::Conj;
-    if (tag == "art" || tag == "article" || tag == "det" || tag == "determiner") return PosClass::Det;
-    if (tag == "num" || tag == "number") return PosClass::Num;
-    if (tag == "interj" || tag == "interjection") return PosClass::Interj;
-    return PosClass::Unknown;
-}
-
-static bool has_pos_class(const std::vector<std::string> &tags, PosClass cls){
-    for (const auto &t : tags){
-        if (pos_class_from_tag(t) == cls) return true;
-    }
-    return false;
-}
-
-static const std::vector<std::string> &dictionary_pos_for_token(const std::string &surface){
-    static const std::vector<std::string> empty;
-    auto key = normalize_dictionary_key(surface);
-    if (key.empty()) return empty;
-
-    auto it = global_pos_cache.find(key);
-    return (it == global_pos_cache.end()) ? empty : it->second;
-}
-
-static bool first_alpha_is_upper(const std::string &s){
-    for (char c : s){
-        unsigned char uc = static_cast<unsigned char>(c);
-        if (std::isalpha(uc) != 0) return std::isupper(uc) != 0;
-    }
-    return false;
-}
-
-static bool first_alpha_is_lower(const std::string &s){
-    for (char c : s){
-        unsigned char uc = static_cast<unsigned char>(c);
-        if (std::isalpha(uc) != 0) return std::islower(uc) != 0;
-    }
-    return false;
-}
-
-static bool is_sentence_boundary_token(const std::string &s){
-    if (s.empty()) return false;
-    char c = s.back();
-    return c == '.' || c == '!' || c == '?';
-}
-
-static bool is_open_punct_token(const std::string &s){
-    return s == "(" || s == "[" || s == "{" || s == "\"" || s == "'";
-}
-
-static bool is_punctuation_only_token(const std::string &s){
-    if (s.empty()) return false;
-    for (char c : s){
-        unsigned char uc = static_cast<unsigned char>(c);
-        if (std::isalnum(uc) != 0) return false;
-    }
-    return true;
-}
-
-static std::string to_lower_ascii(std::string s) {
-    std::transform(s.begin(), s.end(), s.begin(),
-                   [](unsigned char c) { return static_cast<char>(std::tolower(c)); });
-    return s;
-}
-
-static bool is_common_determiner(const std::string &s) {
-    static const std::unordered_set<std::string> words = {
-        "a", "an", "the", "this", "that", "these", "those",
-        "my", "your", "his", "her", "its", "our", "their",
-        "all", "some", "any", "each", "every", "either", "neither",
-        "no", "many", "much", "more", "most", "few", "fewer", "less",
-        "little", "several", "both", "another", "other", "such",
-        "own", "same", "certain", "what", "which", "whose",
-        "whatever", "whichever", "enough", "various", "particular"
-    };
-
-    return words.find(to_lower_ascii(s)) != words.end();
-}
-
-static bool is_common_preposition(const std::string &s) {
-    static const std::unordered_set<std::string> words = {
-        "of", "in", "on", "at", "by", "for", "from", "with",
-        "into", "onto", "about", "over", "under", "after", "before",
-        "between", "through", "during", "without", "within", "across",
-        "against", "among", "around", "as", "like", "per", "via",
-        "toward", "towards", "upon", "beneath", "beside", "besides",
-        "beyond", "inside", "outside", "near", "past", "since",
-        "until", "till", "up", "down", "off", "out", "underneath",
-        "amid", "amidst", "despite", "except", "regarding", "concerning",
-        "throughout", "amongst", "opposite", "along", "behind", "ahead",
-        "above", "below", "plus", "minus"
-    };
-
-    return words.find(to_lower_ascii(s)) != words.end();
-}
-
-static bool is_common_aux_or_modal(const std::string &s) {
-    static const std::unordered_set<std::string> words = {
-        "to", "be", "am", "is", "are", "was", "were", "been", "being",
-        "have", "has", "had", "do", "does", "did",
-        "can", "could", "may", "might", "must", "shall", "should", "will",
-        "would", "ought", "need", "dare", "used",
-        "cannot", "can't", "won't", "wouldn't", "shouldn't", "couldn't",
-        "mustn't", "mayn't", "mightn't", "shan't"
-    };
-
-    return words.find(to_lower_ascii(s)) != words.end();
-}
-
-static bool begins_with_vowel_sound(const std::string &s){
-    if (s.empty()) return false;
-
-    const std::string t = to_lower_ascii(s);
-
-    static const std::unordered_set<std::string> vowel_sound_exceptions = {
-        "hour", "hours", "honest", "honesty", "honor", "honour", "honorable", "honourable",
-        "heir", "heiress", "herb", "herbal", "herbalist", "homage", "hono(u)rary"
-    };
-
-    static const std::unordered_set<std::string> consonant_sound_prefixes = {
-        "uni", "univ", "unit", "use", "user", "euro", "eu", "one", "once", "ouija",
-        "ufo", "ut", "uk", "ubiq", "ewe", "eul", "eup", "x", "y"
-    };
-
-    for (const auto &w : vowel_sound_exceptions) {
-        if (t.rfind(w, 0) == 0) {
-            return true;
-        }
-    }
-
-    for (const auto &p : consonant_sound_prefixes) {
-        if (t.rfind(p, 0) == 0) {
-            return false;
-        }
-    }
-
-    if (t.empty()) return false;
-    char c = t[0];
-    return c == 'a' || c == 'e' || c == 'i' || c == 'o' || c == 'u';
-}
-
-static double english_rule_bonus(const std::string &context_tok, const std::string &cand){
-    const std::string ctx_key = normalize_dictionary_key(context_tok);
-    const std::string cand_key = normalize_dictionary_key(cand);
-
-    const auto &ctx_tags = dictionary_pos_for_token(context_tok);
-    const auto &cand_tags = dictionary_pos_for_token(cand);
-
-    const bool sentence_start = context_tok.empty() || is_sentence_boundary_token(context_tok) || is_open_punct_token(context_tok);
-
-    const bool cand_nounish = has_pos_class(cand_tags, PosClass::Noun) ||
-                               has_pos_class(cand_tags, PosClass::Adj) ||
-                               has_pos_class(cand_tags, PosClass::Pron) ||
-                               has_pos_class(cand_tags, PosClass::Num);
-
-    const bool cand_verbish = has_pos_class(cand_tags, PosClass::Verb);
-    const bool cand_advish  = has_pos_class(cand_tags, PosClass::Adv);
-    const bool cand_prepish = has_pos_class(cand_tags, PosClass::Prep);
-    const bool cand_detish  = has_pos_class(cand_tags, PosClass::Det);
-
-    double bonus = 0.0;
-
-    if (!cand_key.empty()){
-        if (sentence_start){
-            bonus += first_alpha_is_upper(cand) ? 0.22 : -0.08;
-        } else if (first_alpha_is_upper(cand)){
-            bonus -= 0.03;
-        }
-    }
-
-    if (ctx_key == "a" || ctx_key == "an"){
-        const bool vowel = begins_with_vowel_sound(cand_key.empty() ? cand : cand_key);
-        bonus += ((ctx_key == "an") == vowel) ? 0.28 : -0.18;
-    }
-
-    const bool ctx_det = has_pos_class(ctx_tags, PosClass::Det) || is_common_determiner(ctx_key);
-    const bool ctx_prep = has_pos_class(ctx_tags, PosClass::Prep) || is_common_preposition(ctx_key);
-    const bool ctx_aux = is_common_aux_or_modal(ctx_key);
-
-    if (ctx_det){
-        if (cand_nounish) bonus += 0.20;
-        if (cand_verbish || cand_advish || cand_prepish) bonus -= 0.08;
-    }
-
-    if (ctx_prep){
-        if (cand_nounish) bonus += 0.16;
-        if (cand_verbish) bonus -= 0.06;
-    }
-
-    if (ctx_aux){
-        if (cand_verbish) bonus += 0.18;
-        if (cand_detish) bonus -= 0.04;
-    }
-
-    if (has_pos_class(ctx_tags, PosClass::Pron) || has_pos_class(ctx_tags, PosClass::Noun)){
-        if (cand_verbish) bonus += 0.05;
-    }
-
-    if (!context_tok.empty() && (context_tok.back() == ',' || context_tok.back() == ';' || context_tok.back() == ':')){
-        if (!cand.empty() && first_alpha_is_lower(cand)) bonus += 0.04;
-    }
-
-    if (is_punctuation_only_token(cand)){
-        if (sentence_start) bonus -= 0.05;
-        else if (!context_tok.empty() && std::isalnum(static_cast<unsigned char>(context_tok.back())) != 0) bonus += 0.03;
-    }
-
-    if (is_sentence_boundary_token(cand)) bonus += 0.06;
-
-    return bonus;
-}
-
-static std::vector<std::string> tokenize_whitespace(const std::string &s){
-    std::istringstream iss(s);
-    std::vector<std::string> out;
-    std::string t;
-    while (iss >> t) out.push_back(t);
-    return out;
-}
-
-// Static helper to check dictionary presence using the existing pos cache
-static inline bool check_dictionary(const std::string &word) {
-    return !dictionary_pos_for_token(word).empty();
-}
-
-// Morphological lemmatizer to convert sub-tokens to their dictionary-defined base form.
-// Returns std::optional<std::string>: contains the valid base form if found in the dictionary,
-// or std::nullopt if the token has no definition and should be ignored.
-static std::optional<std::string> get_valid_base_form(const std::string &word) {
-    if (word.empty()) return std::nullopt;
-
-    // 1. BEFORE extracting/stemming, check if the original form is already in the dictionary
-    if (check_dictionary(word)) {
-        return word;
-    }
-
-    // 2. Continuous verb form (-ing)
-    if (word.length() > 3 && word.substr(word.length() - 3) == "ing") {
-        // e.g., playing -> play
-        std::string base1 = word.substr(0, word.length() - 3);
-        if (check_dictionary(base1)) return base1;
-
-        // e.g., making -> make
-        std::string base2 = base1 + "e";
-        if (check_dictionary(base2)) return base2;
-
-        // e.g., running -> run (doubled consonant check)
-        if (base1.length() >= 2 && base1.back() == base1[base1.length() - 2]) {
-            std::string base3 = base1.substr(0, base1.length() - 1);
-            if (check_dictionary(base3)) return base3;
-        }
-    }
-
-    // 3. Past tense / Participle form (-ed)
-    if (word.length() > 2 && word.substr(word.length() - 2) == "ed") {
-        // e.g., baked -> bake
-        std::string base1 = word.substr(0, word.length() - 1);
-        if (check_dictionary(base1)) return base1;
-
-        // e.g., played -> play
-        std::string base2 = word.substr(0, word.length() - 2);
-        if (check_dictionary(base2)) return base2;
-
-        // e.g., hopped -> hop (doubled consonant check)
-        if (base2.length() >= 2 && base2.back() == base2[base2.length() - 2]) {
-            std::string base3 = base2.substr(0, base2.length() - 1);
-            if (check_dictionary(base3)) return base3;
-        }
-    }
-
-    // 4. Superlative form (-est)
-    if (word.length() > 3 && word.substr(word.length() - 3) == "est") {
-        // e.g., simplest -> simple (strip 'st')
-        std::string base1 = word.substr(0, word.length() - 2);
-        if (check_dictionary(base1)) return base1;
-
-        // e.g., fastest -> fast
-        std::string base2 = word.substr(0, word.length() - 3);
-        if (check_dictionary(base2)) return base2;
-
-        // e.g., happiest -> happy (-iest -> -y)
-        if (word.length() > 4 && word.substr(word.length() - 4) == "iest") {
-            std::string base_y = word.substr(0, word.length() - 4) + "y";
-            if (check_dictionary(base_y)) return base_y;
-        }
-
-        // e.g., biggest -> big (doubled consonant check)
-        if (base2.length() >= 2 && base2.back() == base2[base2.length() - 2]) {
-            std::string base3 = base2.substr(0, base2.length() - 1);
-            if (check_dictionary(base3)) return base3;
-        }
-    }
-
-    // 5. Comparative form (-er)
-    if (word.length() > 2 && word.substr(word.length() - 2) == "er") {
-        // e.g., simpler -> simple (strip 'r')
-        std::string base1 = word.substr(0, word.length() - 1);
-        if (check_dictionary(base1)) return base1;
-
-        // e.g., faster -> fast
-        std::string base2 = word.substr(0, word.length() - 2);
-        if (check_dictionary(base2)) return base2;
-
-        // e.g., happier -> happy (-ier -> -y)
-        if (word.length() > 3 && word.substr(word.length() - 3) == "ier") {
-            std::string base_y = word.substr(0, word.length() - 3) + "y";
-            if (check_dictionary(base_y)) return base_y;
-        }
-
-        // e.g., bigger -> big (doubled consonant check)
-        if (base2.length() >= 2 && base2.back() == base2[base2.length() - 2]) {
-            std::string base3 = base2.substr(0, base2.length() - 1);
-            if (check_dictionary(base3)) return base3;
-        }
-    }
-
-    // 6. Adverbial form (-ly)
-    if (word.length() > 2 && word.substr(word.length() - 2) == "ly") {
-        // e.g., quickly -> quick
-        std::string base1 = word.substr(0, word.length() - 2);
-        if (check_dictionary(base1)) return base1;
-
-        // e.g., happily -> happy (-ily -> -y)
-        if (word.length() > 3 && word.substr(word.length() - 3) == "ily") {
-            std::string base2 = word.substr(0, word.length() - 3) + "y";
-            if (check_dictionary(base2)) return base2;
-        }
-
-        // e.g., basically -> basic (-ally -> -ic)
-        if (word.length() > 4 && word.substr(word.length() - 4) == "ally") {
-            std::string base3 = word.substr(0, word.length() - 4);
-            if (check_dictionary(base3)) return base3;
-
-            std::string base4 = base3 + "al";
-            if (check_dictionary(base4)) return base4;
-        }
-
-        // e.g., gently -> gentle (-ly -> -le)
-        if (word.back() == 'y' && word.length() > 2 && word[word.length() - 2] == 'l') {
-            std::string base5 = word.substr(0, word.length() - 1) + "e";
-            if (check_dictionary(base5)) return base5;
-        }
-    }
-
-    // 7. Noun suffix (-ness)
-    if (word.length() > 4 && word.substr(word.length() - 4) == "ness") {
-        // e.g., sadness -> sad
-        std::string base1 = word.substr(0, word.length() - 4);
-        if (check_dictionary(base1)) return base1;
-
-        // e.g., happiness -> happy (-iness -> -y)
-        if (base1.length() > 1 && base1.back() == 'i') {
-            std::string base2 = base1.substr(0, base1.length() - 1) + "y";
-            if (check_dictionary(base2)) return base2;
-        }
-    }
-
-    // 8. Noun suffix (-ment)
-    if (word.length() > 4 && word.substr(word.length() - 4) == "ment") {
-        // e.g., development -> develop
-        std::string base1 = word.substr(0, word.length() - 4);
-        if (check_dictionary(base1)) return base1;
-    }
-
-    // 9. Plural / 3rd-person singular form (-ies -> -y)
-    if (word.length() > 3 && word.substr(word.length() - 3) == "ies") {
-        std::string base = word.substr(0, word.length() - 3) + "y";
-        if (check_dictionary(base)) return base;
-    }
-
-    // 10. Plural / 3rd-person singular form (-ves -> -f / -fe)
-    if (word.length() > 3 && word.substr(word.length() - 3) == "ves") {
-        std::string base1 = word.substr(0, word.length() - 3) + "f";
-        if (check_dictionary(base1)) return base1;
-
-        std::string base2 = word.substr(0, word.length() - 3) + "fe";
-        if (check_dictionary(base2)) return base2;
-    }
-
-    // 11. Plural form (-es)
-    if (word.length() > 2 && word.substr(word.length() - 2) == "es") {
-        std::string base = word.substr(0, word.length() - 2);
-        if (check_dictionary(base)) return base;
-    }
-
-    // 12. Plural form (-s)
-    if (word.length() > 1 && word.back() == 's' && word[word.length() - 2] != 's') {
-        std::string base = word.substr(0, word.length() - 1);
-        if (check_dictionary(base)) return base;
-    }
-
-    // No dictionary-defined base form found
-    return std::nullopt;
-}
-
-// Helper arrays for number-to-words
-static const char* const ones[] = {"", "one", "two", "three", "four", "five", "six", "seven", "eight", "nine",
-                                   "ten", "eleven", "twelve", "thirteen", "fourteen", "fifteen", "sixteen",
-                                   "seventeen", "eighteen", "nineteen"};
-static const char* const tens[] = {"", "", "twenty", "thirty", "forty", "fifty", "sixty", "seventy", "eighty", "ninety"};
-static const char* const scales[] = {"", "thousand", "million", "billion", "trillion", "quadrillion",
-                                     "quintillion", "sextillion", "septillion", "octillion", "nonillion",
-                                     "decillion", "undecillion", "duodecillion", "tredecillion",
-                                     "quattuordecillion", "quindecillion", "sexdecillion", "septendecillion",
-                                     "octodecillion", "novemdecillion", "vigintillion", "googol"};
-
-static std::vector<std::string> digit_to_words(const std::string& input_s) {
-    std::vector<std::string> words;
-    if (input_s.empty()) return words;
-
-    // Split the number into integer and fractional parts
-    std::string int_part = input_s;
-    std::string frac_part = "";
-    size_t dot_pos = input_s.find('.');
-    if (dot_pos != std::string::npos) {
-        int_part = input_s.substr(0, dot_pos);
-        frac_part = input_s.substr(dot_pos + 1);
-    }
-
-    // Clean the integer part by removing commas
-    std::string s;
-    for (char c : int_part) {
-        if (std::isdigit(static_cast<unsigned char>(c))) {
-            s.push_back(c);
-        }
-    }
-
-    // Process the integer part
-    if (s.empty()) {
-        // If there are no integer digits, ensure there are fractional digits before parsing
-        bool has_frac_digits = false;
-        for (char c : frac_part) {
-            if (std::isdigit(static_cast<unsigned char>(c))) has_frac_digits = true;
-        }
-        if (!has_frac_digits) return words;
-        words.push_back("zero");
-    } else {
-        size_t first_non_zero = s.find_first_not_of('0');
-        if (first_non_zero == std::string::npos) {
-            words.push_back("zero");
-        } else {
-            std::string num = s.substr(first_non_zero);
-
-            // Fallback for extremely large numbers to prevent overflow
-            if (num.length() > 69) {
-                for (char c : num) {
-                    words.push_back(ones[c - '0']);
-                }
-            } else {
-                int scale_idx = 0;
-                while (!num.empty()) {
-                    int chunk_len = std::min<int>(3, num.length());
-                    std::string chunk_str = num.substr(num.length() - chunk_len);
-                    num = num.substr(0, num.length() - chunk_len);
-
-                    int chunk = std::stoi(chunk_str);
-                    if (chunk > 0) {
-                        std::vector<std::string> chunk_words;
-                        int h = chunk / 100;
-                        int remainder = chunk % 100;
-
-                        if (h > 0) {
-                            chunk_words.push_back(ones[h]);
-                            chunk_words.push_back("hundred");
-                        }
-
-                        if (remainder > 0) {
-                            if (remainder < 20) {
-                                chunk_words.push_back(ones[remainder]);
-                            } else {
-                                chunk_words.push_back(tens[remainder / 10]);
-                                if (remainder % 10 > 0) {
-                                    chunk_words.push_back(ones[remainder % 10]);
-                                }
-                            }
-                        }
-
-                        if (scale_idx > 0 && scale_idx < static_cast<int>(sizeof(scales)/sizeof(scales[0]))) {
-                            chunk_words.push_back(scales[scale_idx]);
-                        }
-
-                        words.insert(words.begin(), chunk_words.begin(), chunk_words.end());
-                    }
-                    scale_idx++;
-                }
-            }
-        }
-    }
-
-    // Process the fractional part
-    if (!frac_part.empty()) {
-        bool has_frac_digits = false;
-        for (char c : frac_part) {
-            if (std::isdigit(static_cast<unsigned char>(c))) has_frac_digits = true;
-        }
-        if (has_frac_digits) {
-            words.push_back("point");
-            for (char c : frac_part) {
-                if (std::isdigit(static_cast<unsigned char>(c))) {
-                    words.push_back(ones[c - '0']);
-                }
-            }
-        }
-    }
-
-    return words;
-}
-
-static std::vector<std::string> tokenize_others(const std::string &s) {
-    if (check_dictionary(s)) {
-        std::string lower_s;
-        lower_s.reserve(s.size());
-        for (char c : s) {
-            lower_s.push_back(to_low(c));
-        }
-        return { lower_s };
-    }
-
-    // Check if the entire string is a formatted number (pure digits, commas, and dots)
-    auto is_num_char = [](unsigned char c) {
-        return std::isdigit(c) != 0 || c == ',' || c == '.';
-    };
-    bool is_all_digits = !s.empty() && std::all_of(s.begin(), s.end(), is_num_char) &&
-                         std::any_of(s.begin(), s.end(), [](unsigned char c) { return std::isdigit(c) != 0; });
-
-    if (is_all_digits) {
-        return digit_to_words(s);
-    }
-
-    std::vector<std::string> out;
-    std::string cur;
-
-    auto flush = [&]() {
-        if (!cur.empty()) {
-            // Check numbers inside mixed tokens (e.g. "word1,234.56")
-            bool cur_all_digits = std::all_of(cur.begin(), cur.end(), [](unsigned char c) {
-                return std::isdigit(c) != 0 || c == ',' || c == '.';
-            });
-            bool has_digits = std::any_of(cur.begin(), cur.end(), [](unsigned char c) {
-                return std::isdigit(c) != 0;
-            });
-
-            if (cur_all_digits && has_digits) {
-                auto words = digit_to_words(cur);
-                out.insert(out.end(), words.begin(), words.end());
-            } else {
-                auto valid_base = get_valid_base_form(cur);
-                if (valid_base.has_value()) {
-                    out.push_back(valid_base.value());
-                }
-            }
-            cur.clear();
-        }
-    };
-
-    for (size_t i = 0; i < s.size(); ++i) {
-        unsigned char uc = static_cast<unsigned char>(s[i]);
-        char ch = static_cast<char>(uc);
-
-        // Check for Operator/Symbol extraction
-        if (std::ispunct(uc) || (std::isprint(uc) && !std::isalnum(uc) && !std::isspace(uc))) {
-
-            // Bypass regular punctuation extraction inside a formatted number
-            if ((ch == ',' || ch == '.') &&
-                !cur.empty() &&
-                (std::isdigit(static_cast<unsigned char>(cur.back())) != 0 || cur.back() == ',' || cur.back() == '.') &&
-                (i + 1 < s.size()) && std::isdigit(static_cast<unsigned char>(s[i+1])) != 0) {
-                cur.push_back(ch);
-                continue;
-            }
-
-            flush();
-            std::string sym(1, ch);
-
-            // Check for double-character operators
-            if (i + 1 < s.size()) {
-                unsigned char nxt_uc = static_cast<unsigned char>(s[i+1]);
-                if (std::ispunct(nxt_uc) || (std::isprint(nxt_uc) && !std::isalnum(nxt_uc) && !std::isspace(nxt_uc))) {
-                    std::string sym2 = sym + static_cast<char>(nxt_uc);
-                    if (check_dictionary(sym2)) {
-                        out.push_back(sym2);
-                        ++i;
-                        continue;
-                    }
-                }
-            }
-            if (check_dictionary(sym)) {
-                out.push_back(sym);
-            }
-            continue;
-        }
-
-        if (std::isspace(uc)) {
-            flush();
-            continue;
-        }
-
-        const bool is_upper = std::isupper(uc) != 0;
-        bool camel_boundary = false;
-        if (is_upper && !cur.empty()) {
-            unsigned char prev = static_cast<unsigned char>(s[i - 1]);
-            const bool prev_lower_or_digit = (std::islower(prev) != 0) || (std::isdigit(prev) != 0);
-            const bool prev_upper = std::isupper(prev) != 0;
-            const bool next_lower = (i + 1 < s.size()) && (std::islower(static_cast<unsigned char>(s[i + 1])) != 0);
-            camel_boundary = prev_lower_or_digit || (prev_upper && next_lower);
-        }
-
-        bool digit_boundary = false;
-        if (!cur.empty()) {
-            unsigned char prev = static_cast<unsigned char>(s[i - 1]);
-            bool prev_digit = (std::isdigit(prev) != 0) || prev == ',' || prev == '.';
-            bool curr_digit = std::isdigit(uc) != 0;
-            if (prev_digit != curr_digit) {
-                digit_boundary = true;
-            }
-        }
-
-        if (camel_boundary || digit_boundary) {
-            flush();
-        }
-
-        if (std::isalnum(uc) != 0) {
-            cur.push_back(static_cast<char>(std::tolower(uc)));
-        } else {
-            flush();
-        }
-    }
-    flush();
-    return out;
-}
-
-using StrPtr = const std::string*;
-using TokenId = std::uint32_t;
-static constexpr TokenId TOKEN_ID_INVALID = 0xFFFFFFFFu;
-
-static inline std::size_t popcount_u64(std::uint64_t x){
-#if defined(_MSC_VER)
-    return static_cast<std::size_t>(__popcnt64(x));
-#else
-    return static_cast<std::size_t>(__builtin_popcountll(static_cast<unsigned long long>(x)));
-#endif
-}
-
-struct StringInterner {
-    std::unordered_map<std::string, TokenId> pool;
-    std::vector<StrPtr> string_by_id;
-    mutable std::mutex m;
-
-    StrPtr intern(const std::string &s){
-        std::lock_guard<std::mutex> lk(m);
-
-        const TokenId id = static_cast<TokenId>(string_by_id.size());
-        auto [it, inserted] = pool.emplace(s, id);
-        if (inserted) string_by_id.push_back(&it->first);
-        return &it->first;
-    }
-
-    TokenId id_of(const std::string &s) const {
-        std::lock_guard<std::mutex> lk(m);
-        auto it = pool.find(s);
-        return (it == pool.end()) ? TOKEN_ID_INVALID : it->second;
-    }
-
-    TokenId id_of(StrPtr p) const {
-        return p ? id_of(*p) : TOKEN_ID_INVALID;
-    }
-
-    StrPtr ptr_from_id(TokenId id) const {
-        std::lock_guard<std::mutex> lk(m);
-        return (id < string_by_id.size()) ? string_by_id[static_cast<size_t>(id)] : nullptr;
-    }
-
-    std::size_t size() const {
-        std::lock_guard<std::mutex> lk(m);
-        return string_by_id.size();
-    }
-
-    void reset(std::size_t reserve_count){
-        std::lock_guard<std::mutex> lk(m);
-        pool.clear();
-        pool.reserve(reserve_count);
-        string_by_id.clear();
-        string_by_id.reserve(reserve_count);
-    }
-
-    std::vector<StrPtr> snapshot_ptrs() const {
-        std::lock_guard<std::mutex> lk(m);
-        std::vector<StrPtr> out;
-        out.reserve(pool.size());
-        for (const auto &pr : pool) out.push_back(&pr.first);
-        return out;
-    }
-};
-
-static inline void bitset_set(std::uint64_t *bits, std::size_t words, TokenId id){
-    if (id == TOKEN_ID_INVALID) return;
-    const std::size_t idx = static_cast<std::size_t>(id >> 6);
-    if (idx >= words) return;
-    bits[idx] |= (1ULL << (id & 63u));
-}
-
-static inline std::size_t bitset_count(const std::uint64_t *bits, std::size_t words){
-    std::size_t total = 0;
-    for (std::size_t i = 0; i < words; ++i) total += popcount_u64(bits[i]);
-    return total;
-}
-
-static inline std::size_t bitset_intersection_count(const std::uint64_t *a, const std::uint64_t *b, std::size_t words){
-    std::size_t total = 0;
-    for (std::size_t i = 0; i < words; ++i) total += popcount_u64(a[i] & b[i]);
-    return total;
-}
-
-static void build_def_tokens_cache(){
-    global_def_tokens_cache.clear();
-    global_pos_cache.clear();
-
-    const int num_entries = static_cast<int>(global_dictionary_entries.size());
-
-    // Pass 1: build the POS cache first. tokenize_others() and the
-    // morphological base-form logic depend on this cache.
-    #pragma omp parallel
-    {
-        std::unordered_map<std::string, std::vector<std::string>> local_pos_cache;
-
-        #pragma omp for schedule(guided) nowait
-        for (int i = 0; i < num_entries; ++i){
-            const auto &entry = global_dictionary_entries[static_cast<size_t>(i)];
-            const std::string pos = normalize_pos_tag(entry.pos);
-            if (pos.empty()) continue;
-
-            std::string current_sub;
-            auto add_pos = [&](const std::string &word){
-                const std::string key = normalize_dictionary_key(word);
-                if (!key.empty()) local_pos_cache[key].push_back(pos);
-            };
-
-            for (char c : entry.word){
-                if (c == ';' || c == ','){
-                    if (!current_sub.empty()){
-                        add_pos(current_sub);
-                        current_sub.clear();
-                    }
-                } else {
-                    current_sub.push_back(c);
-                }
-            }
-            if (!current_sub.empty()) add_pos(current_sub);
-        }
-
-        #pragma omp critical
-        {
-            for (auto &pr : local_pos_cache){
-                auto &vec = global_pos_cache[pr.first];
-                vec.insert(vec.end(),
-                           std::make_move_iterator(pr.second.begin()),
-                           std::make_move_iterator(pr.second.end()));
-            }
-        }
-    }
-
-    std::vector<std::vector<std::string>*> pos_values;
-    pos_values.reserve(global_pos_cache.size());
-    for (auto &pr : global_pos_cache) pos_values.push_back(&pr.second);
-
-    #pragma omp parallel for schedule(dynamic)
-    for (ptrdiff_t i = 0; i < static_cast<ptrdiff_t>(pos_values.size()); ++i){
-        auto &v = *pos_values[static_cast<size_t>(i)];
-        std::sort(v.begin(), v.end());
-        v.erase(std::unique(v.begin(), v.end()), v.end());
-    }
-
-    // Pass 2: POS lookup is now valid, so definition tokenization can safely
-    // use dictionary lookup and morphology.
-    #pragma omp parallel
-    {
-        std::unordered_map<std::string, std::vector<std::string>> local_def_cache;
-
-        #pragma omp for schedule(guided) nowait
-        for (int i = 0; i < num_entries; ++i){
-            const auto &entry = global_dictionary_entries[static_cast<size_t>(i)];
-
-            std::vector<std::string> all_def_toks;
-            for (const auto &def : entry.definitions){
-                auto toks = tokenize_others(def);
-                all_def_toks.insert(all_def_toks.end(),
-                                    std::make_move_iterator(toks.begin()),
-                                    std::make_move_iterator(toks.end()));
-            }
-
-            if (all_def_toks.empty()) continue;
-
-            std::string current_sub;
-            auto add_defs = [&](const std::string &word){
-                const std::string key = normalize_dictionary_key(word);
-                if (key.empty()) return;
-                auto &defs = local_def_cache[key];
-                defs.insert(defs.end(), all_def_toks.begin(), all_def_toks.end());
-            };
-
-            for (char c : entry.word){
-                if (c == ';' || c == ','){
-                    if (!current_sub.empty()){
-                        add_defs(current_sub);
-                        current_sub.clear();
-                    }
-                } else {
-                    current_sub.push_back(c);
-                }
-            }
-            if (!current_sub.empty()) add_defs(current_sub);
-        }
-
-        #pragma omp critical
-        {
-            for (auto &pr : local_def_cache){
-                auto &vec = global_def_tokens_cache[pr.first];
-                vec.insert(vec.end(),
-                           std::make_move_iterator(pr.second.begin()),
-                           std::make_move_iterator(pr.second.end()));
-            }
-        }
-    }
-
-    std::vector<std::vector<std::string>*> def_values;
-    def_values.reserve(global_def_tokens_cache.size());
-    for (auto &pr : global_def_tokens_cache) def_values.push_back(&pr.second);
-
-    #pragma omp parallel for schedule(dynamic)
-    for (ptrdiff_t i = 0; i < static_cast<ptrdiff_t>(def_values.size()); ++i){
-        auto &v = *def_values[static_cast<size_t>(i)];
-        std::sort(v.begin(), v.end());
-        v.erase(std::unique(v.begin(), v.end()), v.end());
-    }
-}
-
-using NextSet = std::vector<StrPtr>;
-
-struct NgramHash {
-    std::size_t operator()(const std::vector<StrPtr>& v) const noexcept {
-        std::size_t seed = v.size();
-        for(auto& p : v) {
-            seed ^= std::hash<StrPtr>()(p) + 0x9e3779b9 + (seed << 6) + (seed >> 2);
-        }
-        return seed;
-    }
-};
-
-struct KnowledgeBase {
-    StringInterner interner;
-    // Modified to use a vector of StrPtr (N-gram) as the key
-    std::unordered_map<std::vector<StrPtr>, NextSet, NgramHash> next;
-    mutable std::mutex m;
-
-    std::unordered_map<StrPtr, std::vector<StrPtr>> def_index;
-    mutable std::mutex def_m;
-    int def_depth = 0;
-
-    void add_ngram(const std::vector<StrPtr>& ctx, StrPtr v){
-        for (auto p : ctx) ensure_def_for_interned(p);
-        ensure_def_for_interned(v);
-
-        std::lock_guard<std::mutex> lk(m);
-        auto &vec = next[ctx];
-        for (auto p : vec) if (p == v) return;
-        vec.push_back(v);
-    }
-
-    void set_def_depth(int D){
-        std::lock_guard<std::mutex> lk(def_m);
-        if (D != def_depth){
-            def_index.clear();
-            def_depth = D;
-        }
-    }
-
-    void ensure_def_for_interned(StrPtr wp){
-        if (wp == nullptr || def_depth <= 0) return;
-
-        {
-            std::lock_guard<std::mutex> lk(def_m);
-            if (def_index.find(wp) != def_index.end()) return;
-        }
-
-        std::unordered_set<StrPtr> acc;
-        std::vector<StrPtr> frontier;
-
-        const std::string start_key = normalize_dictionary_key(*wp);
-        if (!start_key.empty()){
-            auto it_def = global_def_tokens_cache.find(start_key);
-            if (it_def != global_def_tokens_cache.end()){
-                for (const auto &tok : it_def->second){
-                    StrPtr tp = interner.intern(tok);
-                    if (acc.insert(tp).second) frontier.push_back(tp);
-                }
-            }
-        }
-
-        for (int depth = 1; depth < def_depth && !frontier.empty(); ++depth){
-            std::vector<StrPtr> next_frontier;
-
-            for (StrPtr w : frontier){
-                const std::string key = normalize_dictionary_key(*w);
-                if (key.empty()) continue;
-
-                auto it2 = global_def_tokens_cache.find(key);
-                if (it2 == global_def_tokens_cache.end()) continue;
-
-                for (const auto &tok : it2->second){
-                    StrPtr tp = interner.intern(tok);
-                    if (acc.insert(tp).second) next_frontier.push_back(tp);
-                }
-            }
-
-            frontier.swap(next_frontier);
-        }
-
-        std::vector<StrPtr> out;
-        out.reserve(acc.size());
-        for (StrPtr p : acc) out.push_back(p);
-
-        {
-            std::lock_guard<std::mutex> lk(def_m);
-            def_index.emplace(wp, std::move(out));
-        }
-    }
-
-    struct NextSetView {
-        std::unique_lock<std::mutex> lock;
-        const NextSet *value = nullptr;
-
-        NextSetView(std::unique_lock<std::mutex> &&lk, const NextSet *v)
-            : lock(std::move(lk)), value(v) {}
-
-        NextSetView(NextSetView &&) noexcept = default;
-        NextSetView &operator=(NextSetView &&) noexcept = default;
-        NextSetView(const NextSetView &) = delete;
-        NextSetView &operator=(const NextSetView &) = delete;
-
-        const NextSet &get() const noexcept { return *value; }
-    };
-
-    std::optional<NextSetView> lookup_ngram(const std::vector<StrPtr>& ctx) const {
-        std::unique_lock<std::mutex> lk(m);
-        auto it = next.find(ctx);
-        if (it == next.end()) return std::nullopt;
-        return NextSetView(std::move(lk), &it->second);
-    }
-};
-
-static void learn_tokens_ngram(KnowledgeBase &kb, const std::vector<std::string>& tokens, int n_gram_size) {
-    std::vector<StrPtr> window;
-    for (const auto& tok : tokens) {
-        StrPtr tp = kb.interner.intern(tok);
-        for (size_t j = 0; j < window.size(); ++j) {
-            std::vector<StrPtr> ctx(window.begin() + j, window.end());
-            kb.add_ngram(ctx, tp);
-        }
-        window.push_back(tp);
-        if (window.size() > static_cast<size_t>(n_gram_size)) window.erase(window.begin());
-    }
-}
-
-static std::vector<StrPtr>
-intern_tokens(KnowledgeBase &kb, const std::vector<std::string> &tokens)
-{
-    std::vector<StrPtr> out;
-    out.reserve(tokens.size());
-    for (const auto &t : tokens) out.push_back(kb.interner.intern(t));
-    return out;
-}
-
-static inline bool json_valid_index(size_t i, size_t n){ return i < n; }
-
-static std::string parse_quoted_string(const std::string &text, size_t &i){
-    std::string out;
-    if (!json_valid_index(i, text.size()) || text[i] != '"') throw std::runtime_error("expected '\"'");
-    ++i;
-    while (json_valid_index(i, text.size())){
-        char c = text[i++];
-        if (c == '"') break;
-        if (c == '\\'){
-            if (!json_valid_index(i, text.size())) break;
-            char e = text[i++];
-            if (e=='n') out.push_back('\n');
-            else if (e=='t') out.push_back('\t');
-            else out.push_back(e);
-        } else out.push_back(c);
-    }
-    return out;
-}
-
-static void skip_spaces(const std::string &s, size_t &i){
-    while (json_valid_index(i, s.size()) && is_space(s[i])) ++i;
-}
-
-static void skip_json_value(const std::string &s, size_t &i);
-
-static std::vector<std::string> parse_json_string_array(const std::string &text, size_t &i){
-    std::vector<std::string> out;
-    if (!json_valid_index(i, text.size()) || text[i] != '[') return out;
-
-    ++i;
-    while (true){
-        skip_spaces(text, i);
-        if (!json_valid_index(i, text.size())) break;
-        if (text[i] == ']'){ ++i; break; }
-
-        if (text[i] == '"') out.push_back(parse_quoted_string(text, i));
-        else skip_json_value(text, i);
-
-        skip_spaces(text, i);
-        if (json_valid_index(i, text.size()) && text[i] == ','){ ++i; continue; }
-        if (json_valid_index(i, text.size()) && text[i] == ']'){ ++i; break; }
-    }
-
-    return out;
-}
-
-static void skip_json_value(const std::string &s, size_t &i){
-    skip_spaces(s, i);
-    if (!json_valid_index(i, s.size())) return;
-
-    if (s[i] == '"'){
-        (void)parse_quoted_string(s, i);
-        return;
-    }
-
-    if (s[i] == '['){
-        ++i;
-        while (json_valid_index(i, s.size())){
-            skip_spaces(s, i);
-            if (!json_valid_index(i, s.size())) break;
-            if (s[i] == ']'){ ++i; break; }
-            skip_json_value(s, i);
-            skip_spaces(s, i);
-            if (json_valid_index(i, s.size()) && s[i] == ','){ ++i; continue; }
-            if (json_valid_index(i, s.size()) && s[i] == ']'){ ++i; break; }
-        }
-        return;
-    }
-
-    if (s[i] == '{'){
-        ++i;
-        while (json_valid_index(i, s.size())){
-            skip_spaces(s, i);
-            if (!json_valid_index(i, s.size())) break;
-            if (s[i] == '}'){ ++i; break; }
-            if (s[i] == '"'){
-                (void)parse_quoted_string(s, i);
-                skip_spaces(s, i);
-                if (json_valid_index(i, s.size()) && s[i] == ':') ++i;
-                skip_json_value(s, i);
-                skip_spaces(s, i);
-                if (json_valid_index(i, s.size()) && s[i] == ','){ ++i; continue; }
-                if (json_valid_index(i, s.size()) && s[i] == '}'){ ++i; break; }
-            } else {
-                ++i;
-            }
-        }
-        return;
-    }
-
-    while (json_valid_index(i, s.size())){
-        char c = s[i];
-        if (c == ',' || c == ']' || c == '}' || is_space(c)) break;
-        ++i;
-    }
-}
-
-static std::vector<DictionaryEntry> parse_dictionary_json(){
-    std::vector<DictionaryEntry> dictionary;
-    if (dictionary_json_len == 0) return dictionary;
-
-    std::string text;
-    text.reserve(dictionary_json_len);
-    for (unsigned int b = 0; b < dictionary_json_len; ++b){
-        text.push_back(static_cast<char>(dictionary_json[b]));
-    }
-
-    size_t i = 0;
-    skip_spaces(text, i);
-    if (!json_valid_index(i, text.size()) || text[i] != '[') return dictionary;
-    ++i;
-
-    while (true){
-        skip_spaces(text, i);
-        if (!json_valid_index(i, text.size())) break;
-        if (text[i] == ']'){ ++i; break; }
-        if (text[i] != '{'){
-            skip_json_value(text, i);
-            skip_spaces(text, i);
-            if (json_valid_index(i, text.size()) && text[i] == ','){ ++i; continue; }
-            if (json_valid_index(i, text.size()) && text[i] == ']'){ ++i; break; }
-            continue;
-        }
-
-        ++i;
-        DictionaryEntry entry;
-
-        while (true){
-            skip_spaces(text, i);
-            if (!json_valid_index(i, text.size())) break;
-            if (text[i] == '}'){ ++i; break; }
-
-            std::string field = parse_quoted_string(text, i);
-            skip_spaces(text, i);
-            if (!json_valid_index(i, text.size()) || text[i] != ':') break;
-            ++i;
-            skip_spaces(text, i);
-
-            if (field == "word"){
-                entry.word = parse_quoted_string(text, i);
-            } else if (field == "pos"){
-                entry.pos = parse_quoted_string(text, i);
-            } else if (field == "definitions"){
-                entry.definitions = parse_json_string_array(text, i);
-            } else {
-                skip_json_value(text, i);
-            }
-
-            skip_spaces(text, i);
-            if (json_valid_index(i, text.size()) && text[i] == ','){ ++i; continue; }
-            if (json_valid_index(i, text.size()) && text[i] == '}'){ ++i; break; }
-        }
-
-        if (!entry.word.empty()) dictionary.push_back(std::move(entry));
-
-        skip_spaces(text, i);
-        if (json_valid_index(i, text.size()) && text[i] == ','){ ++i; continue; }
-        if (json_valid_index(i, text.size()) && text[i] == ']'){ ++i; break; }
-    }
-
-    return dictionary;
-}
-
-static void add_semantic_fingerprint(
-    StrPtr token,
-    const std::unordered_map<StrPtr, std::vector<StrPtr>> &def_index,
-    const StringInterner &interner,
-    std::uint64_t *bits,
-    std::size_t words)
-{
-    if (!token) return;
-
-    TokenId tid = interner.id_of(token);
-    if (tid != TOKEN_ID_INVALID) bitset_set(bits, words, tid);
-
-    auto it = def_index.find(token);
-    if (it != def_index.end()){
-        for (StrPtr d : it->second){
-            TokenId did = interner.id_of(d);
-            if (did != TOKEN_ID_INVALID) bitset_set(bits, words, did);
-        }
-    }
-
-    const std::vector<std::string> sub_tokens = tokenize_others(*token);
-    for (const std::string &sub_tok : sub_tokens){
-        const TokenId sub_tid = interner.id_of(sub_tok);
-        if (sub_tid == TOKEN_ID_INVALID) continue;
-
-        bitset_set(bits, words, sub_tid);
-
-        const StrPtr sub_ptr = interner.ptr_from_id(sub_tid);
-        if (!sub_ptr) continue;
-
-        auto sub_it = def_index.find(sub_ptr);
-        if (sub_it == def_index.end()) continue;
-
-        for (StrPtr d : sub_it->second){
-            TokenId did = interner.id_of(d);
-            if (did != TOKEN_ID_INVALID) bitset_set(bits, words, did);
-        }
-    }
-}
-
-static StrPtr best_candidate_by_similarity(
-    const NextSet &cands,
-    const std::vector<StrPtr> &context,
-    const std::unordered_map<StrPtr, std::vector<StrPtr>> &def_index,
-    const StringInterner &interner,
-    const std::unordered_map<std::string,int> &recent_counts,
-    double repeat_penalty,
-    const std::string &context_tok)
-{
-    if (cands.empty()) return nullptr;
-    if (cands.size() == 1) return cands[0];
-
-    const std::size_t words = std::max<std::size_t>(1, (interner.size() + 63u) / 64u);
-    std::vector<std::uint64_t> agg_words(words, 0ULL);
-
-    for (StrPtr t : context){
-        add_semantic_fingerprint(t, def_index, interner, agg_words.data(), words);
-    }
-
-    const std::size_t agg_count = bitset_count(agg_words.data(), words);
-    const std::size_t M = cands.size();
-
-    std::vector<std::uint64_t> cand_words(M * words, 0ULL);
-    std::vector<std::size_t> cand_counts(M, 0);
-
-    for (std::size_t i = 0; i < M; ++i){
-        std::uint64_t *row = cand_words.data() + i * words;
-        add_semantic_fingerprint(cands[i], def_index, interner, row, words);
-        cand_counts[i] = bitset_count(row, words);
-    }
-
-    std::vector<double> scores(M, 0.0);
-
-#if defined(_OPENMP) && defined(CHATIPC_ENABLE_OMP_TARGET)
-    const bool use_target = (omp_get_num_devices() > 0) && (M >= 256);
-#else
-    const bool use_target = false;
-#endif
-
-    if (use_target){
-        std::uint64_t *agg_ptr = agg_words.data();
-        std::uint64_t *cand_ptr = cand_words.data();
-        std::size_t *count_ptr = cand_counts.data();
-        double *score_ptr = scores.data();
-        const std::size_t cand_words_total = cand_words.size();
-
-        #pragma omp target data map(to: agg_ptr[0:words], cand_ptr[0:cand_words_total], count_ptr[0:M]) map(from: score_ptr[0:M])
-        {
-            #pragma omp target teams distribute parallel for
-            for (ptrdiff_t i = 0; i < static_cast<ptrdiff_t>(M); ++i){
-                const std::uint64_t *row = cand_ptr + static_cast<std::size_t>(i) * words;
-
-                std::size_t inter = 0;
-                for (std::size_t w = 0; w < words; ++w){
-                    inter += popcount_u64(agg_ptr[w] & row[w]);
-                }
-
-                const std::size_t uni = agg_count + count_ptr[static_cast<size_t>(i)] - inter;
-                score_ptr[static_cast<size_t>(i)] =
-                    uni ? static_cast<double>(inter) / static_cast<double>(uni) : 0.0;
-            }
-        }
-    } else {
-        #ifdef _OPENMP
-        #pragma omp parallel for schedule(static)
-        #endif
-        for (ptrdiff_t i = 0; i < static_cast<ptrdiff_t>(M); ++i){
-            const std::size_t idx = static_cast<std::size_t>(i);
-            const std::uint64_t *row = cand_words.data() + idx * words;
-            const std::size_t inter = bitset_intersection_count(agg_words.data(), row, words);
-            const std::size_t uni = agg_count + cand_counts[idx] - inter;
-            scores[idx] = uni ? static_cast<double>(inter) / static_cast<double>(uni) : 0.0;
-        }
-    }
-
-    double best = -1e9;
-    StrPtr best_tok = nullptr;
-
-    for (std::size_t i = 0; i < M; ++i){
-        const std::string &tok = *cands[i];
-        const std::string tok_key = normalize_dictionary_key(tok);
-        const std::string count_key = tok_key.empty() ? tok : tok_key;
-
-        auto rc_it = recent_counts.find(count_key);
-        const int cnt = (rc_it == recent_counts.end()) ? 0 : rc_it->second;
-
-        const double adjusted =
-            scores[i] +
-            english_rule_bonus(context_tok, tok) -
-            repeat_penalty * static_cast<double>(cnt);
-
-        if (adjusted > best ||
-            (adjusted == best && best_tok != nullptr && tok < *best_tok)){
-            best = adjusted;
-            best_tok = cands[i];
-        }
-    }
-
-    return best_tok;
-}
-
-static std::vector<std::string> construct_response(KnowledgeBase &kb,
-                                                   const std::vector<std::string> &prompt_toks,
-                                                   size_t response_maxlen,
-                                                   double repeat_penalty,
-                                                   int n_gram_size,
-                                                   bool allow_path_loops)
-{
-    std::vector<std::string> resp;
-    if (prompt_toks.empty() || response_maxlen == 0) return resp;
-
-    std::vector<StrPtr> context = intern_tokens(kb, prompt_toks);
-    std::unordered_map<std::string, int> recent_counts;
-
-    auto would_create_2_cycle = [&](const std::string &cand) -> bool {
-        if (resp.size() < 3) return false;
-        return normalize_dictionary_key(cand) == normalize_dictionary_key(resp[resp.size() - 2]) &&
-               normalize_dictionary_key(resp.back()) == normalize_dictionary_key(resp[resp.size() - 3]);
-    };
-
-    for (size_t step = 0; step < response_maxlen; ++step){
-        std::optional<KnowledgeBase::NextSetView> candidate_view;
-        const std::string &context_tok = *context.back();
-
-        int current_n = std::min(static_cast<int>(context.size()), n_gram_size);
-        while (current_n > 0){
-            std::vector<StrPtr> search_ctx(context.end() - current_n, context.end());
-            auto view = kb.lookup_ngram(search_ctx);
-            if (view && !view->get().empty()){
-                candidate_view = std::move(view);
-                break;
-            }
-            --current_n;
-        }
-
-        if (!candidate_view) break;
-        const NextSet &candidates = candidate_view->get();
-
-        if (candidates.size() == 1){
-            const StrPtr ptr = candidates[0];
-            const std::string &only = *ptr;
-            const std::string only_key = normalize_dictionary_key(only);
-            const std::string &count_key = only_key.empty() ? only : only_key;
-
-            auto rc_it = recent_counts.find(count_key);
-            if (!allow_path_loops && rc_it != recent_counts.end() && rc_it->second > 0) break;
-
-            resp.push_back(only);
-            context.push_back(ptr);
-            ++recent_counts[count_key];
-            std::cout << only << ' ' << std::flush;
-            continue;
-        }
-
-        const StrPtr chosen_ptr = best_candidate_by_similarity(
-            candidates, context, kb.def_index, kb.interner,
-            recent_counts, repeat_penalty, context_tok
-        );
-
-        if (!chosen_ptr) break;
-        const std::string &chosen = *chosen_ptr;
-        if (would_create_2_cycle(chosen)) break;
-
-        resp.push_back(chosen);
-        context.push_back(chosen_ptr);
-
-        const std::string chosen_key = normalize_dictionary_key(chosen);
-        ++recent_counts[chosen_key.empty() ? chosen : chosen_key];
-
-        std::cout << chosen << ' ' << std::flush;
-    }
-
-    return resp;
-}
-
-static void learn_from_file(KnowledgeBase &kb, const std::string &fname, int n_gram_size){
-    std::ifstream ifs(fname);
-    if (!ifs) return;
-    std::vector<std::string> tokens;
-    std::string tok;
-    while (ifs >> tok) tokens.push_back(tok);
-    learn_tokens_ngram(kb, tokens, n_gram_size);
-}
-
-static void learn_files_parallel(KnowledgeBase &kb, const std::vector<std::string> &files, int n_gram_size){
-    #pragma omp parallel for schedule(dynamic)
-    for (ptrdiff_t i=0;i<static_cast<ptrdiff_t>(files.size());++i){
-        learn_from_file(kb, files[(size_t)i], n_gram_size);
-    }
-}
-
-static constexpr std::uint64_t KB_MAGIC   = 0x434850434B535641ULL;
-static constexpr std::uint64_t KB_VERSION = 2ULL;
-
-static void write_u64(std::ostream &os, std::uint64_t v) {
-    do {
-        unsigned char byte = v & 0x7F;
-        v >>= 7;
-        if (v != 0) byte |= 0x80;
-        os.put(static_cast<char>(byte));
-    } while (v != 0);
-    if (!os) throw std::runtime_error("write_u64 failed");
-}
-
-static std::uint64_t read_u64(std::istream &is) {
-    std::uint64_t v = 0;
-    unsigned int shift = 0;
-    unsigned char byte;
-    do {
-        if (!is.read(reinterpret_cast<char*>(&byte), 1)) {
-            throw std::runtime_error("read_u64 failed");
-        }
-        v |= (static_cast<std::uint64_t>(byte & 0x7F) << shift);
-        shift += 7;
-    } while (byte & 0x80);
-    return v;
-}
-
-static void write_string(std::ostream &os, const std::string &s){
-    write_u64(os, static_cast<std::uint64_t>(s.size()));
-    if (!s.empty()){
-        os.write(s.data(), static_cast<std::streamsize>(s.size()));
-        if(!os) throw std::runtime_error("write_string failed");
-    }
-}
-
-static std::string read_string(std::istream &is){
-    std::uint64_t n = read_u64(is);
-    //if (n > (1ULL << 30)) throw std::runtime_error("save file is corrupted: string too large");
-
-    std::string s;
-    s.resize(static_cast<size_t>(n));
-
-    if (n != 0){
-        is.read(&s[0], static_cast<std::streamsize>(n));
-        if(!is) throw std::runtime_error("read_string failed");
-    }
-    return s;
-}
-
-static void save_kb_binary(const KnowledgeBase &kb, const std::string &fname){
-    const std::string temp = fname + ".tmp";
-
-    {
-        std::ofstream ofs(temp.c_str(), std::ios::binary | std::ios::trunc);
-        if (!ofs) throw std::runtime_error("cannot open temp save file");
-
-        std::vector<StrPtr> pool = kb.interner.snapshot_ptrs();
-        std::sort(pool.begin(), pool.end(),
-                  [](StrPtr a, StrPtr b){ return *a < *b; });
-
-        std::unordered_map<StrPtr, std::uint64_t> id;
-        id.reserve(pool.size());
-        for (std::uint64_t i = 0; i < static_cast<std::uint64_t>(pool.size()); ++i){
-            id.emplace(pool[static_cast<size_t>(i)], i);
-        }
-
-        write_u64(ofs, KB_MAGIC);
-        write_u64(ofs, KB_VERSION);
-        write_u64(ofs, static_cast<std::uint64_t>(kb.def_depth));
-
-        write_u64(ofs, static_cast<std::uint64_t>(pool.size()));
-        for (StrPtr s : pool) write_string(ofs, *s);
-
-        write_u64(ofs, static_cast<std::uint64_t>(kb.next.size()));
-        for (const auto &pr : kb.next){
-            write_u64(ofs, static_cast<std::uint64_t>(pr.first.size()));
-            for (StrPtr ctx_tok : pr.first){
-                write_u64(ofs, id.at(ctx_tok));
-            }
-            write_u64(ofs, static_cast<std::uint64_t>(pr.second.size()));
-            for (StrPtr nxt : pr.second){
-                write_u64(ofs, id.at(nxt));
-            }
-        }
-
-        write_u64(ofs, static_cast<std::uint64_t>(kb.def_index.size()));
-        for (const auto &pr : kb.def_index){
-            write_u64(ofs, id.at(pr.first));
-            write_u64(ofs, static_cast<std::uint64_t>(pr.second.size()));
-            for (StrPtr tok : pr.second){
-                write_u64(ofs, id.at(tok));
-            }
-        }
-
-        ofs.flush();
-        if (!ofs) throw std::runtime_error("failed while writing temp save file");
-    }
-
-    std::remove(fname.c_str());
-    if (std::rename(temp.c_str(), fname.c_str()) != 0){
-        std::remove(temp.c_str());
-        throw std::runtime_error("failed to commit save file");
-    }
-}
-
-static void load_kb_binary(KnowledgeBase &kb, const std::string &fname, int cli_def_depth){
-    std::ifstream ifs(fname, std::ios::binary);
-    if (!ifs) throw std::runtime_error("cannot open load file");
-
-    const std::uint64_t magic = read_u64(ifs);
-    if (magic != KB_MAGIC) throw std::runtime_error("bad save file magic");
-
-    const std::uint64_t version = read_u64(ifs);
-    if (version != KB_VERSION) throw std::runtime_error("unsupported save file version");
-
-    const std::uint64_t file_def_depth = read_u64(ifs);
-
-    const std::uint64_t N = read_u64(ifs);
-    //if (N > (1ULL << 26)) throw std::runtime_error("save file is corrupted: pool too large");
-
-    kb.interner.reset(static_cast<size_t>(N));
-
-    std::vector<StrPtr> ptrs;
-    ptrs.reserve(static_cast<size_t>(N));
-    for (std::uint64_t i = 0; i < N; ++i){
-        std::string s = read_string(ifs);
-        ptrs.push_back(kb.interner.intern(s));
-    }
-
-    // Rebuild next
-    const std::uint64_t E = read_u64(ifs);
-    //if (E > (1ULL << 26)) throw std::runtime_error("save file is corrupted: graph too large");
-
-    {
-        std::lock_guard<std::mutex> lk(kb.m);
-        kb.next.clear();
-        kb.next.reserve(static_cast<size_t>(E));
-    }
-
-    for (std::uint64_t i = 0; i < E; ++i){
-        const std::uint64_t ctx_size = read_u64(ifs);
-        //if (ctx_size > (1ULL << 16)) throw std::runtime_error("save file is corrupted: n-gram context too large");
-
-        std::vector<StrPtr> ctx;
-        ctx.reserve(static_cast<size_t>(ctx_size));
-        for(std::uint64_t c = 0; c < ctx_size; ++c) {
-            const std::uint64_t c_idx = read_u64(ifs);
-            if (c_idx >= ptrs.size()) throw std::runtime_error("save file is corrupted: bad context key");
-            ctx.push_back(ptrs[(size_t)c_idx]);
-        }
-
-        const std::uint64_t M = read_u64(ifs);
-        //if (M > (1ULL << 26)) throw std::runtime_error("save file is corrupted: graph degree too large");
-
-        NextSet vec;
-        vec.reserve(static_cast<size_t>(M));
-
-        for (std::uint64_t j = 0; j < M; ++j){
-            const std::uint64_t v_idx = read_u64(ifs);
-            if (v_idx >= ptrs.size()) throw std::runtime_error("save file is corrupted: bad graph value");
-            vec.push_back(ptrs[(size_t)v_idx]);
-        }
-
-        {
-            std::lock_guard<std::mutex> lk(kb.m);
-            kb.next.emplace(std::move(ctx), std::move(vec));
-        }
-    }
-
-    // Rebuild def_index from file
-    const std::uint64_t K = read_u64(ifs);
-    //if (K > (1ULL << 26)) throw std::runtime_error("save file is corrupted: def_index too large");
-
-    {
-        std::lock_guard<std::mutex> lk(kb.def_m);
-        kb.def_index.clear();
-        kb.def_index.reserve(static_cast<size_t>(K));
-        kb.def_depth = static_cast<int>(file_def_depth);
-    }
-
-    for (std::uint64_t i = 0; i < K; ++i){
-        const std::uint64_t key_idx = read_u64(ifs);
-        const std::uint64_t M = read_u64(ifs);
-
-        if (key_idx >= ptrs.size()) throw std::runtime_error("save file is corrupted: bad def key");
-        //if (M > (1ULL << 26)) throw std::runtime_error("save file is corrupted: def list too large");
-
-        std::vector<StrPtr> toks;
-        toks.reserve(static_cast<size_t>(M));
-
-        for (std::uint64_t j = 0; j < M; ++j){
-            const std::uint64_t v_idx = read_u64(ifs);
-            if (v_idx >= ptrs.size()) throw std::runtime_error("save file is corrupted: bad def value");
-            toks.push_back(ptrs[(size_t)v_idx]);
-        }
-
-        {
-            std::lock_guard<std::mutex> lk(kb.def_m);
-            kb.def_index.emplace(ptrs[(size_t)key_idx], std::move(toks));
-        }
-    }
-
-    if (cli_def_depth != static_cast<int>(file_def_depth)){
-        kb.set_def_depth(cli_def_depth);
-
-        #pragma omp parallel for schedule(dynamic)
-        for (ptrdiff_t i = 0; i < static_cast<ptrdiff_t>(ptrs.size()); ++i){
-            kb.ensure_def_for_interned(ptrs[static_cast<size_t>(i)]);
-        }
-    }
-}
-
-static void print_kb_info(const std::string &fname) {
-    std::ifstream ifs(fname, std::ios::binary);
-    if (!ifs) {
-        std::cerr << "Error: cannot open knowledge-base file " << fname << "\n";
-        return;
-    }
-
-    std::cout << "Reckoning knowledge-base " << fname << " info. ";
-
-    try {
-        const std::uint64_t magic = read_u64(ifs);
-        if (magic != KB_MAGIC) throw std::runtime_error("bad save file magic");
-
-        const std::uint64_t version = read_u64(ifs);
-        if (version != KB_VERSION) throw std::runtime_error("unsupported save file version");
-
-        const std::uint64_t file_def_depth = read_u64(ifs);
-
-        const std::uint64_t N = read_u64(ifs);
-        //if (N > (1ULL << 26)) throw std::runtime_error("save file is corrupted: pool too large");
-
-        // Dynamically read and skip strings since lengths are now variable-length encoded
-        for (std::uint64_t i = 0; i < N; ++i){
-            std::uint64_t len = read_u64(ifs);
-            //if (len > (1ULL << 30)) throw std::runtime_error("save file is corrupted: string too large");
-            if (len > 0) ifs.seekg(static_cast<std::streamoff>(len), std::ios_base::cur);
-        }
-
-        const std::uint64_t E = read_u64(ifs);
-        //if (E > (1ULL << 26)) throw std::runtime_error("save file is corrupted: graph too large");
-
-        std::uint64_t max_ngram = 0;
-        for (std::uint64_t i = 0; i < E; ++i){
-            const std::uint64_t ctx_size = read_u64(ifs);
-            //if (ctx_size > (1ULL << 16)) throw std::runtime_error("save file is corrupted: n-gram context too large");
-
-            if (ctx_size > max_ngram) max_ngram = ctx_size;
-
-            // Sequentially read (and skip) context keys using LEB128 stream parsing
-            for(std::uint64_t c = 0; c < ctx_size; ++c) {
-                read_u64(ifs);
-            }
-
-            const std::uint64_t M = read_u64(ifs);
-            //if (M > (1ULL << 26)) throw std::runtime_error("save file is corrupted: graph degree too large");
-
-            // Sequentially read (and skip) next-node values
-            for(std::uint64_t j = 0; j < M; ++j){
-                read_u64(ifs);
-            }
-        }
-
-        std::cout << "The " << fname << " dictionary depth is " << file_def_depth << " and " << fname << " n-gram max size is " << max_ngram << "\n";
-
-    } catch (const std::exception &e) {
-        std::cerr << "Error reading knowledge-base info: " << e.what() << "\n";
-    }
-}
-
-static void print_commands(const char *p){
-    std::cout << p << " command-line interface options:\n";
-    std::cout << "  --response-max-length N     Set maximum number of token(s) in a response. Default number is 500.\n";
-    std::cout << "  --save-kb FILE              Save the knowledge-base graph to a binary file.\n";
-    std::cout << "  --load-kb FILE              Load a previously saved knowledge-base graph from a binary file.\n";
-    std::cout << "  --info-kb FILE              Show the dictionary-depth and n-gram max size of a knowledge-base file.\n";
-    std::cout << "  --static-kb                 Disable learning prompt(s) and response(s) during interactive session.\n";
-    std::cout << "  --dictionary-depth D        Set depth of definition expansion used during learning. Default depth is 3.\n";
-    std::cout << "  --n-gram-max-size N         Set maximum size of the n-gram where N is the size. Default size is 3.\n";
-    std::cout << "  --learn f1 f2 ...           Learn from text file(s) to update the knowledge-base graph.\n";
-    std::cout << "  --repeat-penalty P          Set penalty for repeated tokens in response. Default penalty is 0.7 (higher penalty reduce repetition).\n";
-    std::cout << "  --allow-path-loops          Allow infinite loops if the knowledge-base graph has a circular single-path reference.\n";
-    std::cout << "  --activate-agi              Activate the Artificial General Intelligence (AGI) features.\n";
-    std::cout << "  --help                      Show " << p << " command-line interface options.\n";
-}
-
-int main(int argc, char **argv){
-    size_t response_maxlen = 500;
-    std::string savefile;
-    std::string load_kb;
-    int def_depth = 3;
-    int n_gram_size = 3;
-    double repeat_penalty = 0.7; // default λ
-    std::vector<std::string> learn_files;
-    bool learn_chat = true;
-    bool allow_path_loops = false;
-
-    for (int i=1;i<argc;++i){
-        std::string a = argv[i];
-        if (a=="--help"){ print_commands(argv[0]); return 0; }
-        if (a=="--response-max-length" && i+1<argc){ response_maxlen = std::stoul(argv[++i]); continue; }
-        if (a=="--save-kb" && i+1<argc){ savefile = argv[++i]; continue; }
-        if (a=="--load-kb" && i+1<argc){ load_kb = argv[++i]; continue; }
-        if (a=="--info-kb" && i+1<argc){ print_kb_info(argv[++i]); return 0; }
-        if (a=="--static-kb"){ learn_chat = false; continue; }
-        if (a=="--dictionary-depth" && i+1<argc){ def_depth = std::stoi(argv[++i]); continue; }
-        if (a=="--n-gram-max-size" && i+1<argc){ n_gram_size = std::max(1, std::stoi(argv[++i])); continue; }
-        if (a=="--learn"){ while(i+1<argc && argv[i+1][0] != '-') learn_files.push_back(argv[++i]); continue; }
-        if (a=="--repeat-penalty" && i+1<argc){ repeat_penalty = std::stod(argv[++i]); continue; }
-        if (a=="--allow-path-loops") { allow_path_loops = true; continue; }
-        if (a=="--activate-agi"){
-                std::cerr << "AGI subscription fee is $1,000 CAD per month. To pay the fee, contact Caleb Nwokocha via email cnwokocha@proton.me\n";
-                return 0;
-        }
-
-        std::cerr << "Error: Unknown or incomplete command-line option '" << a << "'\n";
-        print_commands(argv[0]);
-        return 1;
-    }
-
-    KnowledgeBase kb;
-
-    global_dictionary_entries = parse_dictionary_json();
-    build_def_tokens_cache();
-    kb.set_def_depth(def_depth);
-
-    if (!load_kb.empty()){
-        try { std::cerr << "Loading knowledge base: " << load_kb << "\n";
-        load_kb_binary(kb, load_kb, def_depth); std::cerr << "Loaded knowledge base: " << load_kb << "\n"; }
-        catch (const std::exception &e){ std::cerr << "Error: " << e.what() << "\n"; }
-    }
-
-    if (!learn_files.empty()){
-        std::cerr << "Learning from file/s (" << learn_files.size() << ") using threads=" << omp_get_max_threads() << "\n";
-        learn_files_parallel(kb, learn_files, n_gram_size);
-    }
-
-    std::string line;
-    std::cout << "Ready. Enter prompts.\n";
-    while (std::cout << "> " , std::getline(std::cin, line)){
-        if (line.empty()){ std::cout << "\n"; continue; }
-        auto prompt_toks = tokenize_whitespace(line);
-        if (learn_chat) {learn_tokens_ngram(kb, prompt_toks, n_gram_size);}
-        auto resp = construct_response(kb, prompt_toks, response_maxlen, repeat_penalty, n_gram_size, allow_path_loops);
-        std::cout << "\n";
-        if (!resp.empty() && learn_chat){learn_tokens_ngram(kb, resp, n_gram_size);}
-        if (!savefile.empty()){
-            try { std::cerr << "Saving knowledge base: " << savefile << "\n";
-            save_kb_binary(kb, savefile); std::cerr << "Saved knowledge base: " << savefile << "\n"; }
-            catch (const std::exception &e){ std::cerr << "Error: " << e.what() << "\n"; }
-        }
-    }
-
-    return 0;
-}
+#include <algorithm>
+#include <atomic>
+#include <cctype>
+#include <cinttypes>
+#include <cstring>
+#include <fstream>
+#include <iostream>
+#include <iterator>
+#include <map>
+#include <mutex>
+#include <optional>
+#include <sstream>
+#include <stdexcept>
+#include <string>
+#include <thread>
+#include <unordered_map>
+#include <unordered_set>
+#include <vector>
+
+#ifdef _OPENMP
+#include <omp.h>
+#else
+inline int omp_get_max_threads(){ return 1; }
+inline int omp_get_thread_num(){ return 0; }
+#endif
+
+extern unsigned char dictionary_json[];
+extern unsigned int dictionary_json_len;
+
+
+static inline bool is_space(char c){ return std::isspace(static_cast<unsigned char>(c)) != 0; }
+static inline char to_low(char c){ return static_cast<char>(std::tolower(static_cast<unsigned char>(c))); }
+static inline void safe_flush(std::ostream &os){ os.flush(); }
+
+struct DictionaryEntry {
+    std::string pos;
+    std::string word;
+    std::vector<std::string> definitions;
+};
+
+static std::vector<DictionaryEntry> global_dictionary_entries;
+static std::unordered_map<std::string, std::vector<std::string>> global_def_tokens_cache;
+static std::unordered_map<std::string, std::vector<std::string>> global_pos_cache;
+
+static inline bool is_word_char_for_key(char c){
+    unsigned char uc = static_cast<unsigned char>(c);
+    return std::isalnum(uc) != 0 || c == '\'' || c == '-';
+}
+
+static std::string normalize_dictionary_key(const std::string &s){
+    size_t b = 0, e = s.size();
+    while (b < e && !is_word_char_for_key(s[b])) ++b;
+    while (e > b && !is_word_char_for_key(s[e - 1])) --e;
+
+    std::string out;
+    out.reserve(e - b);
+    for (size_t i = b; i < e; ++i) out.push_back(to_low(s[i]));
+    return out;
+}
+
+static std::string normalize_pos_tag(const std::string &s){
+    std::string out;
+    out.reserve(s.size());
+    for (char c : s){
+        unsigned char uc = static_cast<unsigned char>(c);
+        if (std::isalpha(uc) != 0) out.push_back(to_low(c));
+    }
+    return out;
+}
+
+enum class PosClass {
+    Unknown,
+    Noun,
+    Verb,
+    Adj,
+    Adv,
+    Pron,
+    Prep,
+    Conj,
+    Det,
+    Num,
+    Interj
+};
+
+static PosClass pos_class_from_tag(const std::string &tag){
+    if (tag == "n" || tag == "noun") return PosClass::Noun;
+    if (tag == "v" || tag == "verb" || tag == "part" || tag == "participle" || tag == "p") return PosClass::Verb;
+    if (tag == "a" || tag == "adj" || tag == "adjective") return PosClass::Adj;
+    if (tag == "adv" || tag == "adverb") return PosClass::Adv;
+    if (tag == "pron" || tag == "pronoun") return PosClass::Pron;
+    if (tag == "prep" || tag == "preposition") return PosClass::Prep;
+    if (tag == "conj" || tag == "conjunction") return PosClass::Conj;
+    if (tag == "art" || tag == "article" || tag == "det" || tag == "determiner") return PosClass::Det;
+    if (tag == "num" || tag == "number") return PosClass::Num;
+    if (tag == "interj" || tag == "interjection") return PosClass::Interj;
+    return PosClass::Unknown;
+}
+
+static bool has_pos_class(const std::vector<std::string> &tags, PosClass cls){
+    for (const auto &t : tags){
+        if (pos_class_from_tag(t) == cls) return true;
+    }
+    return false;
+}
+
+static const std::vector<std::string> &dictionary_pos_for_token(const std::string &surface){
+    static const std::vector<std::string> empty;
+    auto key = normalize_dictionary_key(surface);
+    if (key.empty()) return empty;
+
+    auto it = global_pos_cache.find(key);
+    return (it == global_pos_cache.end()) ? empty : it->second;
+}
+
+static bool first_alpha_is_upper(const std::string &s){
+    for (char c : s){
+        unsigned char uc = static_cast<unsigned char>(c);
+        if (std::isalpha(uc) != 0) return std::isupper(uc) != 0;
+    }
+    return false;
+}
+
+static bool first_alpha_is_lower(const std::string &s){
+    for (char c : s){
+        unsigned char uc = static_cast<unsigned char>(c);
+        if (std::isalpha(uc) != 0) return std::islower(uc) != 0;
+    }
+    return false;
+}
+
+static bool is_sentence_boundary_token(const std::string &s){
+    if (s.empty()) return false;
+    char c = s.back();
+    return c == '.' || c == '!' || c == '?';
+}
+
+static bool is_open_punct_token(const std::string &s){
+    return s == "(" || s == "[" || s == "{" || s == "\"" || s == "'";
+}
+
+static bool is_punctuation_only_token(const std::string &s){
+    if (s.empty()) return false;
+    for (char c : s){
+        unsigned char uc = static_cast<unsigned char>(c);
+        if (std::isalnum(uc) != 0) return false;
+    }
+    return true;
+}
+
+static std::string to_lower_ascii(std::string s) {
+    std::transform(s.begin(), s.end(), s.begin(),
+                   [](unsigned char c) { return static_cast<char>(std::tolower(c)); });
+    return s;
+}
+
+static bool is_common_determiner(const std::string &s) {
+    static const std::unordered_set<std::string> words = {
+        "a", "an", "the", "this", "that", "these", "those",
+        "my", "your", "his", "her", "its", "our", "their",
+        "all", "some", "any", "each", "every", "either", "neither",
+        "no", "many", "much", "more", "most", "few", "fewer", "less",
+        "little", "several", "both", "another", "other", "such",
+        "own", "same", "certain", "what", "which", "whose",
+        "whatever", "whichever", "enough", "various", "particular"
+    };
+
+    return words.find(to_lower_ascii(s)) != words.end();
+}
+
+static bool is_common_preposition(const std::string &s) {
+    static const std::unordered_set<std::string> words = {
+        "of", "in", "on", "at", "by", "for", "from", "with",
+        "into", "onto", "about", "over", "under", "after", "before",
+        "between", "through", "during", "without", "within", "across",
+        "against", "among", "around", "as", "like", "per", "via",
+        "toward", "towards", "upon", "beneath", "beside", "besides",
+        "beyond", "inside", "outside", "near", "past", "since",
+        "until", "till", "up", "down", "off", "out", "underneath",
+        "amid", "amidst", "despite", "except", "regarding", "concerning",
+        "throughout", "amongst", "opposite", "along", "behind", "ahead",
+        "above", "below", "plus", "minus"
+    };
+
+    return words.find(to_lower_ascii(s)) != words.end();
+}
+
+static bool is_common_aux_or_modal(const std::string &s) {
+    static const std::unordered_set<std::string> words = {
+        "to", "be", "am", "is", "are", "was", "were", "been", "being",
+        "have", "has", "had", "do", "does", "did",
+        "can", "could", "may", "might", "must", "shall", "should", "will",
+        "would", "ought", "need", "dare", "used",
+        "cannot", "can't", "won't", "wouldn't", "shouldn't", "couldn't",
+        "mustn't", "mayn't", "mightn't", "shan't"
+    };
+
+    return words.find(to_lower_ascii(s)) != words.end();
+}
+
+static bool begins_with_vowel_sound(const std::string &s){
+    if (s.empty()) return false;
+
+    const std::string t = to_lower_ascii(s);
+
+    static const std::unordered_set<std::string> vowel_sound_exceptions = {
+        "hour", "hours", "honest", "honesty", "honor", "honour", "honorable", "honourable",
+        "heir", "heiress", "herb", "herbal", "herbalist", "homage", "hono(u)rary"
+    };
+
+    static const std::unordered_set<std::string> consonant_sound_prefixes = {
+        "uni", "univ", "unit", "use", "user", "euro", "eu", "one", "once", "ouija",
+        "ufo", "ut", "uk", "ubiq", "ewe", "eul", "eup", "x", "y"
+    };
+
+    for (const auto &w : vowel_sound_exceptions) {
+        if (t.rfind(w, 0) == 0) {
+            return true;
+        }
+    }
+
+    for (const auto &p : consonant_sound_prefixes) {
+        if (t.rfind(p, 0) == 0) {
+            return false;
+        }
+    }
+
+    if (t.empty()) return false;
+    char c = t[0];
+    return c == 'a' || c == 'e' || c == 'i' || c == 'o' || c == 'u';
+}
+
+static double english_rule_bonus(const std::string &context_tok, const std::string &cand){
+    const std::string ctx_key = normalize_dictionary_key(context_tok);
+    const std::string cand_key = normalize_dictionary_key(cand);
+
+    const auto &ctx_tags = dictionary_pos_for_token(context_tok);
+    const auto &cand_tags = dictionary_pos_for_token(cand);
+
+    const bool sentence_start = context_tok.empty() || is_sentence_boundary_token(context_tok) || is_open_punct_token(context_tok);
+
+    const bool cand_nounish = has_pos_class(cand_tags, PosClass::Noun) ||
+                               has_pos_class(cand_tags, PosClass::Adj) ||
+                               has_pos_class(cand_tags, PosClass::Pron) ||
+                               has_pos_class(cand_tags, PosClass::Num);
+
+    const bool cand_verbish = has_pos_class(cand_tags, PosClass::Verb);
+    const bool cand_advish  = has_pos_class(cand_tags, PosClass::Adv);
+    const bool cand_prepish = has_pos_class(cand_tags, PosClass::Prep);
+    const bool cand_detish  = has_pos_class(cand_tags, PosClass::Det);
+
+    double bonus = 0.0;
+
+    if (!cand_key.empty()){
+        if (sentence_start){
+            bonus += first_alpha_is_upper(cand) ? 0.22 : -0.08;
+        } else if (first_alpha_is_upper(cand)){
+            bonus -= 0.03;
+        }
+    }
+
+    if (ctx_key == "a" || ctx_key == "an"){
+        const bool vowel = begins_with_vowel_sound(cand_key.empty() ? cand : cand_key);
+        bonus += ((ctx_key == "an") == vowel) ? 0.28 : -0.18;
+    }
+
+    const bool ctx_det = has_pos_class(ctx_tags, PosClass::Det) || is_common_determiner(ctx_key);
+    const bool ctx_prep = has_pos_class(ctx_tags, PosClass::Prep) || is_common_preposition(ctx_key);
+    const bool ctx_aux = is_common_aux_or_modal(ctx_key);
+
+    if (ctx_det){
+        if (cand_nounish) bonus += 0.20;
+        if (cand_verbish || cand_advish || cand_prepish) bonus -= 0.08;
+    }
+
+    if (ctx_prep){
+        if (cand_nounish) bonus += 0.16;
+        if (cand_verbish) bonus -= 0.06;
+    }
+
+    if (ctx_aux){
+        if (cand_verbish) bonus += 0.18;
+        if (cand_detish) bonus -= 0.04;
+    }
+
+    if (has_pos_class(ctx_tags, PosClass::Pron) || has_pos_class(ctx_tags, PosClass::Noun)){
+        if (cand_verbish) bonus += 0.05;
+    }
+
+    if (!context_tok.empty() && (context_tok.back() == ',' || context_tok.back() == ';' || context_tok.back() == ':')){
+        if (!cand.empty() && first_alpha_is_lower(cand)) bonus += 0.04;
+    }
+
+    if (is_punctuation_only_token(cand)){
+        if (sentence_start) bonus -= 0.05;
+        else if (!context_tok.empty() && std::isalnum(static_cast<unsigned char>(context_tok.back())) != 0) bonus += 0.03;
+    }
+
+    if (is_sentence_boundary_token(cand)) bonus += 0.06;
+
+    return bonus;
+}
+
+static std::vector<std::string> tokenize_whitespace(const std::string &s){
+    std::istringstream iss(s);
+    std::vector<std::string> out;
+    std::string t;
+    while (iss >> t) out.push_back(t);
+    return out;
+}
+
+// Static helper to check dictionary presence using the existing pos cache
+static inline bool check_dictionary(const std::string &word) {
+    return !dictionary_pos_for_token(word).empty();
+}
+
+// Morphological lemmatizer to convert sub-tokens to their dictionary-defined base form.
+// Returns std::optional<std::string>: contains the valid base form if found in the dictionary,
+// or std::nullopt if the token has no definition and should be ignored.
+static std::optional<std::string> get_valid_base_form(const std::string &word) {
+    if (word.empty()) return std::nullopt;
+
+    // 1. BEFORE extracting/stemming, check if the original form is already in the dictionary
+    if (check_dictionary(word)) {
+        return word;
+    }
+
+    // 2. Continuous verb form (-ing)
+    if (word.length() > 3 && word.substr(word.length() - 3) == "ing") {
+        // e.g., playing -> play
+        std::string base1 = word.substr(0, word.length() - 3);
+        if (check_dictionary(base1)) return base1;
+
+        // e.g., making -> make
+        std::string base2 = base1 + "e";
+        if (check_dictionary(base2)) return base2;
+
+        // e.g., running -> run (doubled consonant check)
+        if (base1.length() >= 2 && base1.back() == base1[base1.length() - 2]) {
+            std::string base3 = base1.substr(0, base1.length() - 1);
+            if (check_dictionary(base3)) return base3;
+        }
+    }
+
+    // 3. Past tense / Participle form (-ed)
+    if (word.length() > 2 && word.substr(word.length() - 2) == "ed") {
+        // e.g., baked -> bake
+        std::string base1 = word.substr(0, word.length() - 1);
+        if (check_dictionary(base1)) return base1;
+
+        // e.g., played -> play
+        std::string base2 = word.substr(0, word.length() - 2);
+        if (check_dictionary(base2)) return base2;
+
+        // e.g., hopped -> hop (doubled consonant check)
+        if (base2.length() >= 2 && base2.back() == base2[base2.length() - 2]) {
+            std::string base3 = base2.substr(0, base2.length() - 1);
+            if (check_dictionary(base3)) return base3;
+        }
+    }
+
+    // 4. Superlative form (-est)
+    if (word.length() > 3 && word.substr(word.length() - 3) == "est") {
+        // e.g., simplest -> simple (strip 'st')
+        std::string base1 = word.substr(0, word.length() - 2);
+        if (check_dictionary(base1)) return base1;
+
+        // e.g., fastest -> fast
+        std::string base2 = word.substr(0, word.length() - 3);
+        if (check_dictionary(base2)) return base2;
+
+        // e.g., happiest -> happy (-iest -> -y)
+        if (word.length() > 4 && word.substr(word.length() - 4) == "iest") {
+            std::string base_y = word.substr(0, word.length() - 4) + "y";
+            if (check_dictionary(base_y)) return base_y;
+        }
+
+        // e.g., biggest -> big (doubled consonant check)
+        if (base2.length() >= 2 && base2.back() == base2[base2.length() - 2]) {
+            std::string base3 = base2.substr(0, base2.length() - 1);
+            if (check_dictionary(base3)) return base3;
+        }
+    }
+
+    // 5. Comparative form (-er)
+    if (word.length() > 2 && word.substr(word.length() - 2) == "er") {
+        // e.g., simpler -> simple (strip 'r')
+        std::string base1 = word.substr(0, word.length() - 1);
+        if (check_dictionary(base1)) return base1;
+
+        // e.g., faster -> fast
+        std::string base2 = word.substr(0, word.length() - 2);
+        if (check_dictionary(base2)) return base2;
+
+        // e.g., happier -> happy (-ier -> -y)
+        if (word.length() > 3 && word.substr(word.length() - 3) == "ier") {
+            std::string base_y = word.substr(0, word.length() - 3) + "y";
+            if (check_dictionary(base_y)) return base_y;
+        }
+
+        // e.g., bigger -> big (doubled consonant check)
+        if (base2.length() >= 2 && base2.back() == base2[base2.length() - 2]) {
+            std::string base3 = base2.substr(0, base2.length() - 1);
+            if (check_dictionary(base3)) return base3;
+        }
+    }
+
+    // 6. Adverbial form (-ly)
+    if (word.length() > 2 && word.substr(word.length() - 2) == "ly") {
+        // e.g., quickly -> quick
+        std::string base1 = word.substr(0, word.length() - 2);
+        if (check_dictionary(base1)) return base1;
+
+        // e.g., happily -> happy (-ily -> -y)
+        if (word.length() > 3 && word.substr(word.length() - 3) == "ily") {
+            std::string base2 = word.substr(0, word.length() - 3) + "y";
+            if (check_dictionary(base2)) return base2;
+        }
+
+        // e.g., basically -> basic (-ally -> -ic)
+        if (word.length() > 4 && word.substr(word.length() - 4) == "ally") {
+            std::string base3 = word.substr(0, word.length() - 4);
+            if (check_dictionary(base3)) return base3;
+
+            std::string base4 = base3 + "al";
+            if (check_dictionary(base4)) return base4;
+        }
+
+        // e.g., gently -> gentle (-ly -> -le)
+        if (word.back() == 'y' && word.length() > 2 && word[word.length() - 2] == 'l') {
+            std::string base5 = word.substr(0, word.length() - 1) + "e";
+            if (check_dictionary(base5)) return base5;
+        }
+    }
+
+    // 7. Noun suffix (-ness)
+    if (word.length() > 4 && word.substr(word.length() - 4) == "ness") {
+        // e.g., sadness -> sad
+        std::string base1 = word.substr(0, word.length() - 4);
+        if (check_dictionary(base1)) return base1;
+
+        // e.g., happiness -> happy (-iness -> -y)
+        if (base1.length() > 1 && base1.back() == 'i') {
+            std::string base2 = base1.substr(0, base1.length() - 1) + "y";
+            if (check_dictionary(base2)) return base2;
+        }
+    }
+
+    // 8. Noun suffix (-ment)
+    if (word.length() > 4 && word.substr(word.length() - 4) == "ment") {
+        // e.g., development -> develop
+        std::string base1 = word.substr(0, word.length() - 4);
+        if (check_dictionary(base1)) return base1;
+    }
+
+    // 9. Plural / 3rd-person singular form (-ies -> -y)
+    if (word.length() > 3 && word.substr(word.length() - 3) == "ies") {
+        std::string base = word.substr(0, word.length() - 3) + "y";
+        if (check_dictionary(base)) return base;
+    }
+
+    // 10. Plural / 3rd-person singular form (-ves -> -f / -fe)
+    if (word.length() > 3 && word.substr(word.length() - 3) == "ves") {
+        std::string base1 = word.substr(0, word.length() - 3) + "f";
+        if (check_dictionary(base1)) return base1;
+
+        std::string base2 = word.substr(0, word.length() - 3) + "fe";
+        if (check_dictionary(base2)) return base2;
+    }
+
+    // 11. Plural form (-es)
+    if (word.length() > 2 && word.substr(word.length() - 2) == "es") {
+        std::string base = word.substr(0, word.length() - 2);
+        if (check_dictionary(base)) return base;
+    }
+
+    // 12. Plural form (-s)
+    if (word.length() > 1 && word.back() == 's' && word[word.length() - 2] != 's') {
+        std::string base = word.substr(0, word.length() - 1);
+        if (check_dictionary(base)) return base;
+    }
+
+    // No dictionary-defined base form found
+    return std::nullopt;
+}
+
+// Helper arrays for number-to-words
+static const char* const ones[] = {"", "one", "two", "three", "four", "five", "six", "seven", "eight", "nine",
+                                   "ten", "eleven", "twelve", "thirteen", "fourteen", "fifteen", "sixteen",
+                                   "seventeen", "eighteen", "nineteen"};
+static const char* const tens[] = {"", "", "twenty", "thirty", "forty", "fifty", "sixty", "seventy", "eighty", "ninety"};
+static const char* const scales[] = {"", "thousand", "million", "billion", "trillion", "quadrillion",
+                                     "quintillion", "sextillion", "septillion", "octillion", "nonillion",
+                                     "decillion", "undecillion", "duodecillion", "tredecillion",
+                                     "quattuordecillion", "quindecillion", "sexdecillion", "septendecillion",
+                                     "octodecillion", "novemdecillion", "vigintillion", "googol"};
+
+static std::vector<std::string> digit_to_words(const std::string& input_s) {
+    std::vector<std::string> words;
+    if (input_s.empty()) return words;
+
+    // Split the number into integer and fractional parts
+    std::string int_part = input_s;
+    std::string frac_part = "";
+    size_t dot_pos = input_s.find('.');
+    if (dot_pos != std::string::npos) {
+        int_part = input_s.substr(0, dot_pos);
+        frac_part = input_s.substr(dot_pos + 1);
+    }
+
+    // Clean the integer part by removing commas
+    std::string s;
+    for (char c : int_part) {
+        if (std::isdigit(static_cast<unsigned char>(c))) {
+            s.push_back(c);
+        }
+    }
+
+    // Process the integer part
+    if (s.empty()) {
+        // If there are no integer digits, ensure there are fractional digits before parsing
+        bool has_frac_digits = false;
+        for (char c : frac_part) {
+            if (std::isdigit(static_cast<unsigned char>(c))) has_frac_digits = true;
+        }
+        if (!has_frac_digits) return words;
+        words.push_back("zero");
+    } else {
+        size_t first_non_zero = s.find_first_not_of('0');
+        if (first_non_zero == std::string::npos) {
+            words.push_back("zero");
+        } else {
+            std::string num = s.substr(first_non_zero);
+
+            // Fallback for extremely large numbers to prevent overflow
+            if (num.length() > 69) {
+                for (char c : num) {
+                    words.push_back(ones[c - '0']);
+                }
+            } else {
+                int scale_idx = 0;
+                while (!num.empty()) {
+                    int chunk_len = std::min<int>(3, num.length());
+                    std::string chunk_str = num.substr(num.length() - chunk_len);
+                    num = num.substr(0, num.length() - chunk_len);
+
+                    int chunk = std::stoi(chunk_str);
+                    if (chunk > 0) {
+                        std::vector<std::string> chunk_words;
+                        int h = chunk / 100;
+                        int remainder = chunk % 100;
+
+                        if (h > 0) {
+                            chunk_words.push_back(ones[h]);
+                            chunk_words.push_back("hundred");
+                        }
+
+                        if (remainder > 0) {
+                            if (remainder < 20) {
+                                chunk_words.push_back(ones[remainder]);
+                            } else {
+                                chunk_words.push_back(tens[remainder / 10]);
+                                if (remainder % 10 > 0) {
+                                    chunk_words.push_back(ones[remainder % 10]);
+                                }
+                            }
+                        }
+
+                        if (scale_idx > 0 && scale_idx < static_cast<int>(sizeof(scales)/sizeof(scales[0]))) {
+                            chunk_words.push_back(scales[scale_idx]);
+                        }
+
+                        words.insert(words.begin(), chunk_words.begin(), chunk_words.end());
+                    }
+                    scale_idx++;
+                }
+            }
+        }
+    }
+
+    // Process the fractional part
+    if (!frac_part.empty()) {
+        bool has_frac_digits = false;
+        for (char c : frac_part) {
+            if (std::isdigit(static_cast<unsigned char>(c))) has_frac_digits = true;
+        }
+        if (has_frac_digits) {
+            words.push_back("point");
+            for (char c : frac_part) {
+                if (std::isdigit(static_cast<unsigned char>(c))) {
+                    words.push_back(ones[c - '0']);
+                }
+            }
+        }
+    }
+
+    return words;
+}
+
+static std::vector<std::string> tokenize_others(const std::string &s) {
+    if (check_dictionary(s)) {
+        std::string lower_s;
+        lower_s.reserve(s.size());
+        for (char c : s) {
+            lower_s.push_back(to_low(c));
+        }
+        return { lower_s };
+    }
+
+    // Check if the entire string is a formatted number (pure digits, commas, and dots)
+    auto is_num_char = [](unsigned char c) {
+        return std::isdigit(c) != 0 || c == ',' || c == '.';
+    };
+    bool is_all_digits = !s.empty() && std::all_of(s.begin(), s.end(), is_num_char) &&
+                         std::any_of(s.begin(), s.end(), [](unsigned char c) { return std::isdigit(c) != 0; });
+
+    if (is_all_digits) {
+        return digit_to_words(s);
+    }
+
+    std::vector<std::string> out;
+    std::string cur;
+
+    auto flush = [&]() {
+        if (!cur.empty()) {
+            // Check numbers inside mixed tokens (e.g. "word1,234.56")
+            bool cur_all_digits = std::all_of(cur.begin(), cur.end(), [](unsigned char c) {
+                return std::isdigit(c) != 0 || c == ',' || c == '.';
+            });
+            bool has_digits = std::any_of(cur.begin(), cur.end(), [](unsigned char c) {
+                return std::isdigit(c) != 0;
+            });
+
+            if (cur_all_digits && has_digits) {
+                auto words = digit_to_words(cur);
+                out.insert(out.end(), words.begin(), words.end());
+            } else {
+                auto valid_base = get_valid_base_form(cur);
+                if (valid_base.has_value()) {
+                    out.push_back(valid_base.value());
+                }
+            }
+            cur.clear();
+        }
+    };
+
+    for (size_t i = 0; i < s.size(); ++i) {
+        unsigned char uc = static_cast<unsigned char>(s[i]);
+        char ch = static_cast<char>(uc);
+
+        // Check for Operator/Symbol extraction
+        if (std::ispunct(uc) || (std::isprint(uc) && !std::isalnum(uc) && !std::isspace(uc))) {
+
+            // Bypass regular punctuation extraction inside a formatted number
+            if ((ch == ',' || ch == '.') &&
+                !cur.empty() &&
+                (std::isdigit(static_cast<unsigned char>(cur.back())) != 0 || cur.back() == ',' || cur.back() == '.') &&
+                (i + 1 < s.size()) && std::isdigit(static_cast<unsigned char>(s[i+1])) != 0) {
+                cur.push_back(ch);
+                continue;
+            }
+
+            flush();
+            std::string sym(1, ch);
+
+            // Check for double-character operators
+            if (i + 1 < s.size()) {
+                unsigned char nxt_uc = static_cast<unsigned char>(s[i+1]);
+                if (std::ispunct(nxt_uc) || (std::isprint(nxt_uc) && !std::isalnum(nxt_uc) && !std::isspace(nxt_uc))) {
+                    std::string sym2 = sym + static_cast<char>(nxt_uc);
+                    if (check_dictionary(sym2)) {
+                        out.push_back(sym2);
+                        ++i;
+                        continue;
+                    }
+                }
+            }
+            if (check_dictionary(sym)) {
+                out.push_back(sym);
+            }
+            continue;
+        }
+
+        if (std::isspace(uc)) {
+            flush();
+            continue;
+        }
+
+        const bool is_upper = std::isupper(uc) != 0;
+        bool camel_boundary = false;
+        if (is_upper && !cur.empty()) {
+            unsigned char prev = static_cast<unsigned char>(s[i - 1]);
+            const bool prev_lower_or_digit = (std::islower(prev) != 0) || (std::isdigit(prev) != 0);
+            const bool prev_upper = std::isupper(prev) != 0;
+            const bool next_lower = (i + 1 < s.size()) && (std::islower(static_cast<unsigned char>(s[i + 1])) != 0);
+            camel_boundary = prev_lower_or_digit || (prev_upper && next_lower);
+        }
+
+        bool digit_boundary = false;
+        if (!cur.empty()) {
+            unsigned char prev = static_cast<unsigned char>(s[i - 1]);
+            bool prev_digit = (std::isdigit(prev) != 0) || prev == ',' || prev == '.';
+            bool curr_digit = std::isdigit(uc) != 0;
+            if (prev_digit != curr_digit) {
+                digit_boundary = true;
+            }
+        }
+
+        if (camel_boundary || digit_boundary) {
+            flush();
+        }
+
+        if (std::isalnum(uc) != 0) {
+            cur.push_back(static_cast<char>(std::tolower(uc)));
+        } else {
+            flush();
+        }
+    }
+    flush();
+    return out;
+}
+
+using StrPtr = const std::string*;
+using TokenId = std::uint32_t;
+static constexpr TokenId TOKEN_ID_INVALID = 0xFFFFFFFFu;
+
+static inline std::size_t popcount_u64(std::uint64_t x){
+#if defined(_MSC_VER)
+    return static_cast<std::size_t>(__popcnt64(x));
+#else
+    return static_cast<std::size_t>(__builtin_popcountll(static_cast<unsigned long long>(x)));
+#endif
+}
+
+struct StringInterner {
+    std::unordered_set<std::string> pool;
+    std::unordered_map<std::string, TokenId> id_by_string;
+    std::vector<const std::string*> string_by_id;
+    mutable std::mutex m;
+
+    const std::string* intern(const std::string &s){
+        std::lock_guard<std::mutex> lk(m);
+
+        auto [it, inserted] = pool.emplace(s);
+        if (inserted){
+            const TokenId id = static_cast<TokenId>(string_by_id.size());
+            id_by_string.emplace(*it, id);
+            string_by_id.push_back(&*it);
+        }
+
+        return &*it;
+    }
+
+    TokenId id_of(const std::string &s) const {
+        std::lock_guard<std::mutex> lk(m);
+
+        auto it = id_by_string.find(s);
+        return (it == id_by_string.end()) ? TOKEN_ID_INVALID : it->second;
+    }
+
+    TokenId id_of(const std::string *p) const {
+        return p ? id_of(*p) : TOKEN_ID_INVALID;
+    }
+
+    const std::string* ptr_from_id(TokenId id) const {
+        std::lock_guard<std::mutex> lk(m);
+
+        return (id < string_by_id.size()) ? string_by_id[(size_t)id] : nullptr;
+    }
+
+    std::size_t size() const {
+        std::lock_guard<std::mutex> lk(m);
+        return string_by_id.size();
+    }
+};
+
+static inline void bitset_set(std::uint64_t *bits, std::size_t words, TokenId id){
+    if (id == TOKEN_ID_INVALID) return;
+    const std::size_t idx = static_cast<std::size_t>(id >> 6);
+    if (idx >= words) return;
+    bits[idx] |= (1ULL << (id & 63u));
+}
+
+static inline std::size_t bitset_count(const std::uint64_t *bits, std::size_t words){
+    std::size_t total = 0;
+    for (std::size_t i = 0; i < words; ++i) total += popcount_u64(bits[i]);
+    return total;
+}
+
+static inline std::size_t bitset_intersection_count(const std::uint64_t *a, const std::uint64_t *b, std::size_t words){
+    std::size_t total = 0;
+    for (std::size_t i = 0; i < words; ++i) total += popcount_u64(a[i] & b[i]);
+    return total;
+}
+
+static void build_def_tokens_cache(){
+    global_def_tokens_cache.clear();
+    global_pos_cache.clear();
+
+    const int num_entries = static_cast<int>(global_dictionary_entries.size());
+
+    #pragma omp parallel
+    {
+        std::unordered_map<std::string, std::vector<std::string>> local_def_cache;
+        std::unordered_map<std::string, std::vector<std::string>> local_pos_cache;
+
+        #pragma omp for schedule(guided) nowait
+        for (int i = 0; i < num_entries; ++i){
+            const auto &entry = global_dictionary_entries[i];
+
+            std::vector<std::string> all_def_toks;
+            for (const auto &def : entry.definitions){
+                auto toks = tokenize_others(def);
+                all_def_toks.insert(all_def_toks.end(), toks.begin(), toks.end());
+            }
+
+            std::string pos = normalize_pos_tag(entry.pos);
+
+            std::vector<std::string> sub_words;
+            std::string current_sub;
+            for (char c : entry.word) {
+                if (c == ';' || c == ',') {
+                    if (!current_sub.empty()) {
+                        sub_words.push_back(current_sub);
+                        current_sub.clear();
+                    }
+                } else {
+                    current_sub.push_back(c);
+                }
+            }
+            if (!current_sub.empty()) sub_words.push_back(current_sub);
+
+            for (size_t j = 0; j < sub_words.size(); ++j) {
+                const std::string key = normalize_dictionary_key(sub_words[j]);
+                if (key.empty()) continue;
+
+                if (!pos.empty()) {
+                    if (j == sub_words.size() - 1) {
+                        local_pos_cache[key].push_back(std::move(pos));
+                    } else {
+                        local_pos_cache[key].push_back(pos);
+                    }
+                }
+                auto &defs = local_def_cache[key];
+                defs.insert(defs.end(), all_def_toks.begin(), all_def_toks.end());
+            }
+        }
+
+        #pragma omp critical
+        {
+            for (auto &pr : local_def_cache) {
+                auto &vec = global_def_tokens_cache[pr.first];
+                vec.insert(vec.end(), std::make_move_iterator(pr.second.begin()), std::make_move_iterator(pr.second.end()));
+            }
+            for (auto &pr : local_pos_cache) {
+                auto &vec = global_pos_cache[pr.first];
+                vec.insert(vec.end(), std::make_move_iterator(pr.second.begin()), std::make_move_iterator(pr.second.end()));
+            }
+        }
+    }
+
+    // Extract keys to use OpenMP parallel-for deduplication over hash map structures
+    std::vector<std::string> def_keys;
+    def_keys.reserve(global_def_tokens_cache.size());
+    for (const auto &pr : global_def_tokens_cache) def_keys.push_back(pr.first);
+
+    #pragma omp parallel for schedule(dynamic)
+    for (ptrdiff_t i = 0; i < static_cast<ptrdiff_t>(def_keys.size()); ++i){
+        auto &v = global_def_tokens_cache[def_keys[i]];
+        std::sort(v.begin(), v.end());
+        v.erase(std::unique(v.begin(), v.end()), v.end());
+    }
+
+    std::vector<std::string> pos_keys;
+    pos_keys.reserve(global_pos_cache.size());
+    for (const auto &pr : global_pos_cache) pos_keys.push_back(pr.first);
+
+    #pragma omp parallel for schedule(dynamic)
+    for (ptrdiff_t i = 0; i < static_cast<ptrdiff_t>(pos_keys.size()); ++i){
+        auto &v = global_pos_cache[pos_keys[i]];
+        std::sort(v.begin(), v.end());
+        v.erase(std::unique(v.begin(), v.end()), v.end());
+    }
+}
+
+struct PtrHash { size_t operator()(StrPtr p) const noexcept { return std::hash<StrPtr>()(p); } };
+struct PtrEq   { bool operator()(StrPtr a, StrPtr b) const noexcept { return a == b; } };
+
+using NextSet = std::vector<StrPtr>;
+
+struct NgramHash {
+    std::size_t operator()(const std::vector<StrPtr>& v) const noexcept {
+        std::size_t seed = v.size();
+        for(auto& p : v) {
+            seed ^= std::hash<StrPtr>()(p) + 0x9e3779b9 + (seed << 6) + (seed >> 2);
+        }
+        return seed;
+    }
+};
+
+struct KnowledgeBase {
+    StringInterner interner;
+    // Modified to use a vector of StrPtr (N-gram) as the key
+    std::unordered_map<std::vector<StrPtr>, NextSet, NgramHash> next;
+    mutable std::mutex m;
+
+    std::unordered_map<StrPtr, std::vector<StrPtr>, PtrHash, PtrEq> def_index;
+    mutable std::mutex def_m;
+    int def_depth = 0;
+
+    void add_ngram(const std::vector<StrPtr>& ctx, StrPtr v){
+        for (auto p : ctx) ensure_def_for_interned(p);
+        ensure_def_for_interned(v);
+
+        std::lock_guard<std::mutex> lk(m);
+        auto &vec = next[ctx];
+        for (auto p : vec) if (p == v) return;
+        vec.push_back(v);
+    }
+
+    void set_def_depth(int D){
+        std::lock_guard<std::mutex> lk(def_m);
+        if (D != def_depth){
+            def_index.clear();
+            def_depth = D;
+        }
+    }
+
+    void ensure_def_for_interned(StrPtr wp){
+        if (wp == nullptr || def_depth <= 0) return;
+
+        {
+            std::lock_guard<std::mutex> lk(def_m);
+            if (def_index.find(wp) != def_index.end()) return;
+        }
+
+        std::unordered_set<StrPtr, PtrHash, PtrEq> acc;
+        std::vector<StrPtr> frontier;
+
+        const std::string start_key = normalize_dictionary_key(*wp);
+        if (!start_key.empty()){
+            auto it_def = global_def_tokens_cache.find(start_key);
+            if (it_def != global_def_tokens_cache.end()){
+                for (const auto &tok : it_def->second){
+                    StrPtr tp = interner.intern(tok);
+                    if (acc.insert(tp).second) frontier.push_back(tp);
+                }
+            }
+        }
+
+        for (int depth = 1; depth < def_depth && !frontier.empty(); ++depth){
+            std::vector<StrPtr> next_frontier;
+
+            for (StrPtr w : frontier){
+                const std::string key = normalize_dictionary_key(*w);
+                if (key.empty()) continue;
+
+                auto it2 = global_def_tokens_cache.find(key);
+                if (it2 == global_def_tokens_cache.end()) continue;
+
+                for (const auto &tok : it2->second){
+                    StrPtr tp = interner.intern(tok);
+                    if (acc.insert(tp).second) next_frontier.push_back(tp);
+                }
+            }
+
+            frontier.swap(next_frontier);
+        }
+
+        std::vector<StrPtr> out;
+        out.reserve(acc.size());
+        for (StrPtr p : acc) out.push_back(p);
+
+        {
+            std::lock_guard<std::mutex> lk(def_m);
+            def_index.emplace(wp, std::move(out));
+        }
+    }
+
+    std::optional<NextSet> lookup_ngram(const std::vector<StrPtr>& ctx) const {
+        std::lock_guard<std::mutex> lk(m);
+        auto it = next.find(ctx);
+        if (it == next.end()) return std::nullopt;
+        return it->second;
+    }
+};
+
+static void learn_tokens_ngram(KnowledgeBase &kb, const std::vector<std::string>& tokens, int n_gram_size) {
+    std::vector<StrPtr> window;
+    for (const auto& tok : tokens) {
+        StrPtr tp = kb.interner.intern(tok);
+        for (size_t j = 0; j < window.size(); ++j) {
+            std::vector<StrPtr> ctx(window.begin() + j, window.end());
+            kb.add_ngram(ctx, tp);
+        }
+        window.push_back(tp);
+        if (window.size() > static_cast<size_t>(n_gram_size)) window.erase(window.begin());
+    }
+}
+
+static std::vector<StrPtr>
+intern_tokens(KnowledgeBase &kb, const std::vector<std::string> &tokens)
+{
+    std::vector<StrPtr> out;
+    out.reserve(tokens.size());
+    for (const auto &t : tokens) out.push_back(kb.interner.intern(t));
+    return out;
+}
+
+static inline bool json_valid_index(size_t i, size_t n){ return i < n; }
+
+static std::string parse_quoted_string(const std::string &text, size_t &i){
+    std::string out;
+    if (!json_valid_index(i, text.size()) || text[i] != '"') throw std::runtime_error("expected '\"'");
+    ++i;
+    while (json_valid_index(i, text.size())){
+        char c = text[i++];
+        if (c == '"') break;
+        if (c == '\\'){
+            if (!json_valid_index(i, text.size())) break;
+            char e = text[i++];
+            if (e=='n') out.push_back('\n');
+            else if (e=='t') out.push_back('\t');
+            else out.push_back(e);
+        } else out.push_back(c);
+    }
+    return out;
+}
+
+static void skip_spaces(const std::string &s, size_t &i){
+    while (json_valid_index(i, s.size()) && is_space(s[i])) ++i;
+}
+
+static void skip_json_value(const std::string &s, size_t &i);
+
+static std::vector<std::string> parse_json_string_array(const std::string &text, size_t &i){
+    std::vector<std::string> out;
+    if (!json_valid_index(i, text.size()) || text[i] != '[') return out;
+
+    ++i;
+    while (true){
+        skip_spaces(text, i);
+        if (!json_valid_index(i, text.size())) break;
+        if (text[i] == ']'){ ++i; break; }
+
+        if (text[i] == '"') out.push_back(parse_quoted_string(text, i));
+        else skip_json_value(text, i);
+
+        skip_spaces(text, i);
+        if (json_valid_index(i, text.size()) && text[i] == ','){ ++i; continue; }
+        if (json_valid_index(i, text.size()) && text[i] == ']'){ ++i; break; }
+    }
+
+    return out;
+}
+
+static void skip_json_value(const std::string &s, size_t &i){
+    skip_spaces(s, i);
+    if (!json_valid_index(i, s.size())) return;
+
+    if (s[i] == '"'){
+        (void)parse_quoted_string(s, i);
+        return;
+    }
+
+    if (s[i] == '['){
+        ++i;
+        while (json_valid_index(i, s.size())){
+            skip_spaces(s, i);
+            if (!json_valid_index(i, s.size())) break;
+            if (s[i] == ']'){ ++i; break; }
+            skip_json_value(s, i);
+            skip_spaces(s, i);
+            if (json_valid_index(i, s.size()) && s[i] == ','){ ++i; continue; }
+            if (json_valid_index(i, s.size()) && s[i] == ']'){ ++i; break; }
+        }
+        return;
+    }
+
+    if (s[i] == '{'){
+        ++i;
+        while (json_valid_index(i, s.size())){
+            skip_spaces(s, i);
+            if (!json_valid_index(i, s.size())) break;
+            if (s[i] == '}'){ ++i; break; }
+            if (s[i] == '"'){
+                (void)parse_quoted_string(s, i);
+                skip_spaces(s, i);
+                if (json_valid_index(i, s.size()) && s[i] == ':') ++i;
+                skip_json_value(s, i);
+                skip_spaces(s, i);
+                if (json_valid_index(i, s.size()) && s[i] == ','){ ++i; continue; }
+                if (json_valid_index(i, s.size()) && s[i] == '}'){ ++i; break; }
+            } else {
+                ++i;
+            }
+        }
+        return;
+    }
+
+    while (json_valid_index(i, s.size())){
+        char c = s[i];
+        if (c == ',' || c == ']' || c == '}' || is_space(c)) break;
+        ++i;
+    }
+}
+
+static std::vector<DictionaryEntry> parse_dictionary_json(){
+    std::vector<DictionaryEntry> dictionary;
+    if (dictionary_json_len == 0) return dictionary;
+
+    std::string text;
+    text.reserve(dictionary_json_len);
+    for (unsigned int b = 0; b < dictionary_json_len; ++b){
+        text.push_back(static_cast<char>(dictionary_json[b]));
+    }
+
+    size_t i = 0;
+    skip_spaces(text, i);
+    if (!json_valid_index(i, text.size()) || text[i] != '[') return dictionary;
+    ++i;
+
+    while (true){
+        skip_spaces(text, i);
+        if (!json_valid_index(i, text.size())) break;
+        if (text[i] == ']'){ ++i; break; }
+        if (text[i] != '{'){
+            skip_json_value(text, i);
+            skip_spaces(text, i);
+            if (json_valid_index(i, text.size()) && text[i] == ','){ ++i; continue; }
+            if (json_valid_index(i, text.size()) && text[i] == ']'){ ++i; break; }
+            continue;
+        }
+
+        ++i;
+        DictionaryEntry entry;
+
+        while (true){
+            skip_spaces(text, i);
+            if (!json_valid_index(i, text.size())) break;
+            if (text[i] == '}'){ ++i; break; }
+
+            std::string field = parse_quoted_string(text, i);
+            skip_spaces(text, i);
+            if (!json_valid_index(i, text.size()) || text[i] != ':') break;
+            ++i;
+            skip_spaces(text, i);
+
+            if (field == "word"){
+                entry.word = parse_quoted_string(text, i);
+            } else if (field == "pos"){
+                entry.pos = parse_quoted_string(text, i);
+            } else if (field == "definitions"){
+                entry.definitions = parse_json_string_array(text, i);
+            } else {
+                skip_json_value(text, i);
+            }
+
+            skip_spaces(text, i);
+            if (json_valid_index(i, text.size()) && text[i] == ','){ ++i; continue; }
+            if (json_valid_index(i, text.size()) && text[i] == '}'){ ++i; break; }
+        }
+
+        if (!entry.word.empty()) dictionary.push_back(std::move(entry));
+
+        skip_spaces(text, i);
+        if (json_valid_index(i, text.size()) && text[i] == ','){ ++i; continue; }
+        if (json_valid_index(i, text.size()) && text[i] == ']'){ ++i; break; }
+    }
+
+    return dictionary;
+}
+
+static std::string best_candidate_by_similarity(
+    const NextSet &cands,
+    const std::vector<StrPtr> &prompt_ptrs,
+    const std::vector<StrPtr> &resp_ptrs,
+    const std::unordered_map<StrPtr, std::vector<StrPtr>, PtrHash, PtrEq> &def_index,
+    const StringInterner &interner,
+    const std::unordered_map<std::string,int> &recent_counts,
+    double repeat_penalty,
+    const std::string &context_tok)
+{
+    if (cands.empty()) return std::string();
+    if (cands.size() == 1) return *cands[0];
+
+    const std::size_t words = std::max<std::size_t>(1, (interner.size() + 63u) / 64u);
+
+    std::vector<std::uint64_t> agg_words(words, 0ULL);
+
+    auto add_token_and_defs = [&](StrPtr t){
+        if (!t) return;
+
+        // Process the main token and its direct definitions
+        TokenId tid = interner.id_of(t);
+        if (tid != TOKEN_ID_INVALID) bitset_set(agg_words.data(), words, tid);
+
+        auto it = def_index.find(t);
+        if (it != def_index.end()){
+            for (StrPtr d : it->second){
+                TokenId did = interner.id_of(d);
+                if (did != TOKEN_ID_INVALID) bitset_set(agg_words.data(), words, did);
+            }
+        }
+
+        // Tokenize the expanded dictionary string to process sub-tokens
+        std::vector<std::string> sub_tokens = tokenize_others(*t);
+        for (const std::string& sub_tok : sub_tokens) {
+
+            // Use id_of to check if the sub-token exists without adding a new string to the pool
+            TokenId sub_tid = interner.id_of(sub_tok);
+
+            if (sub_tid != TOKEN_ID_INVALID) {
+                // Add the sub-token itself to the bitset
+                bitset_set(agg_words.data(), words, sub_tid);
+
+                // Retrieve the interned pointer and look it up in def_index
+                StrPtr sub_ptr = interner.ptr_from_id(sub_tid);
+                if (sub_ptr) {
+                    auto sub_it = def_index.find(sub_ptr);
+                    if (sub_it != def_index.end()) {
+                        // Add the sub-token's definitions to the bitset
+                        for (StrPtr d : sub_it->second) {
+                            TokenId did = interner.id_of(d);
+                            if (did != TOKEN_ID_INVALID) bitset_set(agg_words.data(), words, did);
+                        }
+                    }
+                }
+            }
+        }
+    };
+
+    for (StrPtr t : prompt_ptrs) add_token_and_defs(t);
+    for (StrPtr t : resp_ptrs) add_token_and_defs(t);
+
+    const std::size_t agg_count = bitset_count(agg_words.data(), words);
+
+    const std::size_t M = cands.size();
+    std::vector<std::uint64_t> cand_words(M * words, 0ULL);
+    std::vector<std::size_t> cand_counts(M, 0);
+
+    for (std::size_t i = 0; i < M; ++i){
+        std::uint64_t *row = cand_words.data() + i * words;
+        const StrPtr cand = cands[i];
+
+        // Tag the candidate word itself
+        TokenId cid = interner.id_of(cand);
+        if (cid != TOKEN_ID_INVALID) bitset_set(row, words, cid);
+
+        // Expand the candidate's direct definition
+        auto it = def_index.find(cand);
+        if (it != def_index.end()){
+            for (StrPtr d : it->second){
+                TokenId did = interner.id_of(d);
+                if (did != TOKEN_ID_INVALID) bitset_set(row, words, did);
+            }
+        }
+
+        // Tokenize the candidate to process sub-tokens (e.g., camelCase, hyphens)
+        std::vector<std::string> sub_tokens = tokenize_others(*cand);
+        for (const std::string& sub_tok : sub_tokens) {
+
+            // Check if the sub-token exists in the knowledge base
+            TokenId sub_tid = interner.id_of(sub_tok);
+
+            if (sub_tid != TOKEN_ID_INVALID) {
+                // Add the sub-token itself to the candidate's semantic fingerprint (row)
+                bitset_set(row, words, sub_tid);
+
+                // Convert the ID back to a pointer
+                StrPtr sub_ptr = interner.ptr_from_id(sub_tid);
+                if (sub_ptr) {
+                    auto sub_it = def_index.find(sub_ptr);
+                    if (sub_it != def_index.end()) {
+                        // Add the sub-token's deeper definitions to the candidate's row
+                        for (StrPtr d : sub_it->second) {
+                            TokenId did = interner.id_of(d);
+                            if (did != TOKEN_ID_INVALID) bitset_set(row, words, did);
+                        }
+                    }
+                }
+            }
+        }
+
+        // Finally, count the bits for the Jaccard similarity math
+        cand_counts[i] = bitset_count(row, words);
+    }
+
+    std::vector<double> scores(M, 0.0);
+
+#if defined(_OPENMP) && defined(CHATIPC_ENABLE_OMP_TARGET)
+    const bool use_target = (omp_get_num_devices() > 0) && (M >= 256);
+#else
+    const bool use_target = false;
+#endif
+
+    if (use_target){
+        std::uint64_t *agg_ptr = agg_words.data();
+        std::uint64_t *cand_ptr = cand_words.data();
+        std::size_t *count_ptr = cand_counts.data();
+        double *score_ptr = scores.data();
+        const std::size_t cand_words_total = cand_words.size();
+
+        #pragma omp target data map(to: agg_ptr[0:words], cand_ptr[0:cand_words_total], count_ptr[0:M]) map(from: score_ptr[0:M])
+        {
+            #pragma omp target teams distribute parallel for
+            for (ptrdiff_t i = 0; i < static_cast<ptrdiff_t>(M); ++i){
+                const std::uint64_t *row = cand_ptr + static_cast<std::size_t>(i) * words;
+
+                std::size_t inter = 0;
+                for (std::size_t w = 0; w < words; ++w){
+                    inter += popcount_u64(agg_ptr[w] & row[w]);
+                }
+
+                const std::size_t uni = agg_count + count_ptr[(size_t)i] - inter;
+                score_ptr[(size_t)i] = uni ? static_cast<double>(inter) / static_cast<double>(uni) : 0.0;
+            }
+        }
+    } else {
+        #ifdef _OPENMP
+        #pragma omp parallel for schedule(static)
+        #endif
+        for (ptrdiff_t i = 0; i < static_cast<ptrdiff_t>(M); ++i){
+            const std::uint64_t *row = cand_words.data() + static_cast<std::size_t>(i) * words;
+
+            const std::size_t inter = bitset_intersection_count(agg_words.data(), row, words);
+            const std::size_t uni = agg_count + cand_counts[(size_t)i] - inter;
+            scores[(size_t)i] = uni ? static_cast<double>(inter) / static_cast<double>(uni) : 0.0;
+        }
+    }
+
+    double best = -1e9;
+    std::string best_tok;
+
+    for (std::size_t i = 0; i < M; ++i){
+        const std::string &tok = *cands[i];
+        const std::string tok_key = normalize_dictionary_key(tok);
+        const std::string count_key = tok_key.empty() ? tok : tok_key;
+
+        auto rc_it = recent_counts.find(count_key);
+        const int cnt = (rc_it == recent_counts.end()) ? 0 : rc_it->second;
+
+        const double adjusted =
+            scores[i] +
+            english_rule_bonus(context_tok, tok) -
+            repeat_penalty * static_cast<double>(cnt);
+
+        if (adjusted > best || (adjusted == best && tok < best_tok)){
+            best = adjusted;
+            best_tok = tok;
+        }
+    }
+
+    return best_tok;
+}
+
+static std::vector<std::string> construct_response(KnowledgeBase &kb,
+                                                   const std::vector<std::string> &prompt_toks,
+                                                   size_t response_maxlen,
+                                                   double repeat_penalty,
+                                                   int n_gram_size,
+                                                   bool allow_path_loops)
+{
+    std::vector<std::string> resp;
+    if (prompt_toks.empty() || response_maxlen == 0) return resp;
+
+    auto prompt_ptrs = intern_tokens(kb, prompt_toks);
+    std::vector<StrPtr> resp_ptrs;
+    std::unordered_map<std::string, int> recent_counts;
+
+    // The context combines prompt and generated text
+    std::vector<StrPtr> context = prompt_ptrs;
+
+    auto would_create_2_cycle = [&](const std::string &cand) -> bool {
+        if (resp.size() < 3) return false;
+        return normalize_dictionary_key(cand) == normalize_dictionary_key(resp[resp.size() - 2]) &&
+               normalize_dictionary_key(resp.back()) == normalize_dictionary_key(resp[resp.size() - 3]);
+    };
+
+    for (size_t step = 0; step < response_maxlen; ++step){
+        NextSet candidates;
+        bool found = false;
+        std::string context_tok;
+
+        int current_n = std::min(static_cast<int>(context.size()), n_gram_size);
+        while (current_n > 0) {
+            std::vector<StrPtr> search_ctx(context.end() - current_n, context.end());
+            auto opt = kb.lookup_ngram(search_ctx);
+            if (opt && !opt->empty()) {
+                candidates = *opt;
+                found = true;
+                context_tok = *search_ctx.back();
+                break;
+            }
+            current_n--;
+        }
+
+        if (!found || candidates.empty()) break;
+
+        if (candidates.size() == 1){
+            std::string only = *candidates[0];
+            std::string only_key = normalize_dictionary_key(only);
+            if (!allow_path_loops && recent_counts[only_key.empty() ? only : only_key] > 0) break;
+
+            resp.push_back(only);
+            StrPtr ptr = kb.interner.intern(only);
+            resp_ptrs.push_back(ptr);
+            context.push_back(ptr);
+            recent_counts[only_key.empty() ? only : only_key] += 1;
+            std::cout << only << ' ' << std::flush;
+            continue;
+        }
+
+        std::string chosen = best_candidate_by_similarity(
+            candidates, prompt_ptrs, resp_ptrs, kb.def_index, kb.interner,
+            recent_counts, repeat_penalty, context_tok
+        );
+
+        if (chosen.empty()) break;
+        if (would_create_2_cycle(chosen)) break;
+
+        resp.push_back(chosen);
+        StrPtr chosen_ptr = kb.interner.intern(chosen);
+        resp_ptrs.push_back(chosen_ptr);
+        context.push_back(chosen_ptr);
+
+        std::string chosen_key = normalize_dictionary_key(chosen);
+        recent_counts[chosen_key.empty() ? chosen : chosen_key] += 1;
+
+        std::cout << chosen << ' ' << std::flush;
+    }
+
+    return resp;
+}
+
+static void learn_from_file(KnowledgeBase &kb, const std::string &fname, int n_gram_size){
+    std::ifstream ifs(fname);
+    if (!ifs) return;
+    std::vector<std::string> tokens;
+    std::string tok;
+    while (ifs >> tok) tokens.push_back(tok);
+    learn_tokens_ngram(kb, tokens, n_gram_size);
+}
+
+static void learn_files_parallel(KnowledgeBase &kb, const std::vector<std::string> &files, int n_gram_size){
+    #pragma omp parallel for schedule(dynamic)
+    for (ptrdiff_t i=0;i<static_cast<ptrdiff_t>(files.size());++i){
+        learn_from_file(kb, files[(size_t)i], n_gram_size);
+    }
+}
+
+static constexpr std::uint64_t KB_MAGIC   = 0x434850434B535641ULL;
+static constexpr std::uint64_t KB_VERSION = 2ULL;
+
+static void write_u64(std::ostream &os, std::uint64_t v) {
+    do {
+        unsigned char byte = v & 0x7F;
+        v >>= 7;
+        if (v != 0) byte |= 0x80;
+        os.put(static_cast<char>(byte));
+    } while (v != 0);
+    if (!os) throw std::runtime_error("write_u64 failed");
+}
+
+static std::uint64_t read_u64(std::istream &is) {
+    std::uint64_t v = 0;
+    unsigned int shift = 0;
+    unsigned char byte;
+    do {
+        if (!is.read(reinterpret_cast<char*>(&byte), 1)) {
+            throw std::runtime_error("read_u64 failed");
+        }
+        v |= (static_cast<std::uint64_t>(byte & 0x7F) << shift);
+        shift += 7;
+    } while (byte & 0x80);
+    return v;
+}
+
+static void write_string(std::ostream &os, const std::string &s){
+    write_u64(os, static_cast<std::uint64_t>(s.size()));
+    if (!s.empty()){
+        os.write(s.data(), static_cast<std::streamsize>(s.size()));
+        if(!os) throw std::runtime_error("write_string failed");
+    }
+}
+
+static std::string read_string(std::istream &is){
+    std::uint64_t n = read_u64(is);
+    //if (n > (1ULL << 30)) throw std::runtime_error("save file is corrupted: string too large");
+
+    std::string s;
+    s.resize(static_cast<size_t>(n));
+
+    if (n != 0){
+        is.read(&s[0], static_cast<std::streamsize>(n));
+        if(!is) throw std::runtime_error("read_string failed");
+    }
+    return s;
+}
+
+static void save_kb_binary(const KnowledgeBase &kb, const std::string &fname){
+    const std::string temp = fname + ".tmp";
+
+    {
+        std::ofstream ofs(temp.c_str(), std::ios::binary | std::ios::trunc);
+        if (!ofs) throw std::runtime_error("cannot open temp save file");
+
+        std::vector<std::string> pool;
+        pool.reserve(kb.interner.pool.size());
+        for (const auto &s : kb.interner.pool) pool.push_back(s);
+
+        std::sort(pool.begin(), pool.end());
+
+        std::unordered_map<std::string, std::uint64_t> id;
+        id.reserve(pool.size());
+        for (std::uint64_t i = 0; i < static_cast<std::uint64_t>(pool.size()); ++i){
+            id.emplace(pool[(size_t)i], i);
+        }
+
+        write_u64(ofs, KB_MAGIC);
+        write_u64(ofs, KB_VERSION);
+        write_u64(ofs, static_cast<std::uint64_t>(kb.def_depth));
+
+        write_u64(ofs, static_cast<std::uint64_t>(pool.size()));
+        for (const auto &s : pool) write_string(ofs, s);
+
+        write_u64(ofs, static_cast<std::uint64_t>(kb.next.size()));
+        for (const auto &pr : kb.next){
+            write_u64(ofs, static_cast<std::uint64_t>(pr.first.size()));
+            for (StrPtr ctx_tok : pr.first){
+                write_u64(ofs, id.at(*ctx_tok));
+            }
+            write_u64(ofs, static_cast<std::uint64_t>(pr.second.size()));
+            for (StrPtr nxt : pr.second){
+                write_u64(ofs, id.at(*nxt));
+            }
+        }
+
+        write_u64(ofs, static_cast<std::uint64_t>(kb.def_index.size()));
+        for (const auto &pr : kb.def_index){
+            write_u64(ofs, id.at(*pr.first));
+            write_u64(ofs, static_cast<std::uint64_t>(pr.second.size()));
+            for (StrPtr tok : pr.second){
+                write_u64(ofs, id.at(*tok));
+            }
+        }
+
+        ofs.flush();
+        if (!ofs) throw std::runtime_error("failed while writing temp save file");
+    }
+
+    std::remove(fname.c_str());
+    if (std::rename(temp.c_str(), fname.c_str()) != 0){
+        std::remove(temp.c_str());
+        throw std::runtime_error("failed to commit save file");
+    }
+}
+
+static void load_kb_binary(KnowledgeBase &kb, const std::string &fname, int cli_def_depth){
+    std::ifstream ifs(fname, std::ios::binary);
+    if (!ifs) throw std::runtime_error("cannot open load file");
+
+    const std::uint64_t magic = read_u64(ifs);
+    if (magic != KB_MAGIC) throw std::runtime_error("bad save file magic");
+
+    const std::uint64_t version = read_u64(ifs);
+    if (version != KB_VERSION) throw std::runtime_error("unsupported save file version");
+
+    const std::uint64_t file_def_depth = read_u64(ifs);
+
+    const std::uint64_t N = read_u64(ifs);
+    //if (N > (1ULL << 26)) throw std::runtime_error("save file is corrupted: pool too large");
+
+    std::vector<std::string> strings;
+    strings.reserve(static_cast<size_t>(N));
+
+    for (std::uint64_t i = 0; i < N; ++i){
+        strings.push_back(read_string(ifs));
+    }
+
+    kb.interner.pool.clear();
+    kb.interner.pool.reserve(static_cast<size_t>(N));
+
+    std::vector<StrPtr> ptrs;
+    ptrs.reserve(static_cast<size_t>(N));
+    for (const auto &s : strings){
+        ptrs.push_back(kb.interner.intern(s));
+    }
+
+    // Rebuild next
+    const std::uint64_t E = read_u64(ifs);
+    //if (E > (1ULL << 26)) throw std::runtime_error("save file is corrupted: graph too large");
+
+    {
+        std::lock_guard<std::mutex> lk(kb.m);
+        kb.next.clear();
+        kb.next.reserve(static_cast<size_t>(E));
+    }
+
+    for (std::uint64_t i = 0; i < E; ++i){
+        const std::uint64_t ctx_size = read_u64(ifs);
+        //if (ctx_size > (1ULL << 16)) throw std::runtime_error("save file is corrupted: n-gram context too large");
+
+        std::vector<StrPtr> ctx;
+        ctx.reserve(static_cast<size_t>(ctx_size));
+        for(std::uint64_t c = 0; c < ctx_size; ++c) {
+            const std::uint64_t c_idx = read_u64(ifs);
+            if (c_idx >= ptrs.size()) throw std::runtime_error("save file is corrupted: bad context key");
+            ctx.push_back(ptrs[(size_t)c_idx]);
+        }
+
+        const std::uint64_t M = read_u64(ifs);
+        //if (M > (1ULL << 26)) throw std::runtime_error("save file is corrupted: graph degree too large");
+
+        NextSet vec;
+        vec.reserve(static_cast<size_t>(M));
+
+        for (std::uint64_t j = 0; j < M; ++j){
+            const std::uint64_t v_idx = read_u64(ifs);
+            if (v_idx >= ptrs.size()) throw std::runtime_error("save file is corrupted: bad graph value");
+            vec.push_back(ptrs[(size_t)v_idx]);
+        }
+
+        {
+            std::lock_guard<std::mutex> lk(kb.m);
+            kb.next.emplace(std::move(ctx), std::move(vec));
+        }
+    }
+
+    // Rebuild def_index from file
+    const std::uint64_t K = read_u64(ifs);
+    //if (K > (1ULL << 26)) throw std::runtime_error("save file is corrupted: def_index too large");
+
+    {
+        std::lock_guard<std::mutex> lk(kb.def_m);
+        kb.def_index.clear();
+        kb.def_index.reserve(static_cast<size_t>(K));
+        kb.def_depth = static_cast<int>(file_def_depth);
+    }
+
+    for (std::uint64_t i = 0; i < K; ++i){
+        const std::uint64_t key_idx = read_u64(ifs);
+        const std::uint64_t M = read_u64(ifs);
+
+        if (key_idx >= ptrs.size()) throw std::runtime_error("save file is corrupted: bad def key");
+        //if (M > (1ULL << 26)) throw std::runtime_error("save file is corrupted: def list too large");
+
+        std::vector<StrPtr> toks;
+        toks.reserve(static_cast<size_t>(M));
+
+        for (std::uint64_t j = 0; j < M; ++j){
+            const std::uint64_t v_idx = read_u64(ifs);
+            if (v_idx >= ptrs.size()) throw std::runtime_error("save file is corrupted: bad def value");
+            toks.push_back(ptrs[(size_t)v_idx]);
+        }
+
+        {
+            std::lock_guard<std::mutex> lk(kb.def_m);
+            kb.def_index.emplace(ptrs[(size_t)key_idx], std::move(toks));
+        }
+    }
+
+    if (cli_def_depth != static_cast<int>(file_def_depth)){
+        kb.set_def_depth(cli_def_depth);
+
+        std::vector<StrPtr> targets;
+        targets.reserve(ptrs.size() + kb.next.size() * 2);
+
+        std::unordered_set<StrPtr, PtrHash, PtrEq> seen;
+        seen.reserve(ptrs.size() + kb.next.size() * 2);
+
+        for (StrPtr p : ptrs){
+            if (seen.insert(p).second) targets.push_back(p);
+        }
+
+        {
+            std::lock_guard<std::mutex> lk(kb.m);
+            for (const auto &pr : kb.next){
+                for (StrPtr ctx_element : pr.first) {
+                    if (seen.insert(ctx_element).second) {
+                        targets.push_back(ctx_element);
+                    }
+                }
+
+                for (StrPtr v : pr.second){
+                    if (seen.insert(v).second) {
+                        targets.push_back(v);
+                    }
+                }
+            }
+        }
+
+        #pragma omp parallel for schedule(dynamic)
+        for (ptrdiff_t i = 0; i < static_cast<ptrdiff_t>(targets.size()); ++i){
+            kb.ensure_def_for_interned(targets[(size_t)i]);
+        }
+    }
+}
+
+static void print_kb_info(const std::string &fname) {
+    std::ifstream ifs(fname, std::ios::binary);
+    if (!ifs) {
+        std::cerr << "Error: cannot open knowledge-base file " << fname << "\n";
+        return;
+    }
+
+    std::cout << "Reckoning knowledge-base " << fname << " info. ";
+
+    try {
+        const std::uint64_t magic = read_u64(ifs);
+        if (magic != KB_MAGIC) throw std::runtime_error("bad save file magic");
+
+        const std::uint64_t version = read_u64(ifs);
+        if (version != KB_VERSION) throw std::runtime_error("unsupported save file version");
+
+        const std::uint64_t file_def_depth = read_u64(ifs);
+
+        const std::uint64_t N = read_u64(ifs);
+        //if (N > (1ULL << 26)) throw std::runtime_error("save file is corrupted: pool too large");
+
+        // Dynamically read and skip strings since lengths are now variable-length encoded
+        for (std::uint64_t i = 0; i < N; ++i){
+            std::uint64_t len = read_u64(ifs);
+            //if (len > (1ULL << 30)) throw std::runtime_error("save file is corrupted: string too large");
+            if (len > 0) ifs.seekg(static_cast<std::streamoff>(len), std::ios_base::cur);
+        }
+
+        const std::uint64_t E = read_u64(ifs);
+        //if (E > (1ULL << 26)) throw std::runtime_error("save file is corrupted: graph too large");
+
+        std::uint64_t max_ngram = 0;
+        for (std::uint64_t i = 0; i < E; ++i){
+            const std::uint64_t ctx_size = read_u64(ifs);
+            //if (ctx_size > (1ULL << 16)) throw std::runtime_error("save file is corrupted: n-gram context too large");
+
+            if (ctx_size > max_ngram) max_ngram = ctx_size;
+
+            // Sequentially read (and skip) context keys using LEB128 stream parsing
+            for(std::uint64_t c = 0; c < ctx_size; ++c) {
+                read_u64(ifs);
+            }
+
+            const std::uint64_t M = read_u64(ifs);
+            //if (M > (1ULL << 26)) throw std::runtime_error("save file is corrupted: graph degree too large");
+
+            // Sequentially read (and skip) next-node values
+            for(std::uint64_t j = 0; j < M; ++j){
+                read_u64(ifs);
+            }
+        }
+
+        std::cout << "The " << fname << " dictionary depth is " << file_def_depth << " and " << fname << " n-gram max size is " << max_ngram << "\n";
+
+    } catch (const std::exception &e) {
+        std::cerr << "Error reading knowledge-base info: " << e.what() << "\n";
+    }
+}
+
+static void print_commands(const char *p){
+    std::cout << p << " command-line interface options:\n";
+    std::cout << "  --response-max-length N     Set maximum number of token(s) in a response. Default number is 500.\n";
+    std::cout << "  --save-kb FILE              Save the knowledge-base graph to a binary file.\n";
+    std::cout << "  --load-kb FILE              Load a previously saved knowledge-base graph from a binary file.\n";
+    std::cout << "  --info-kb FILE              Show the dictionary-depth and n-gram max size of a knowledge-base file.\n";
+    std::cout << "  --static-kb                 Disable learning prompt(s) and response(s) during interactive session.\n";
+    std::cout << "  --dictionary-depth D        Set depth of definition expansion used during learning. Default depth is 3.\n";
+    std::cout << "  --n-gram-max-size N         Set maximum size of the n-gram where N is the size. Default size is 3.\n";
+    std::cout << "  --learn f1 f2 ...           Learn from text file(s) to update the knowledge-base graph.\n";
+    std::cout << "  --repeat-penalty P          Set penalty for repeated tokens in response. Default penalty is 0.7 (higher penalty reduce repetition).\n";
+    std::cout << "  --allow-path-loops          Allow infinite loops if the knowledge-base graph has a circular single-path reference.\n";
+    std::cout << "  --activate-agi              Activate the Artificial General Intelligence (AGI) features.\n";
+    std::cout << "  --help                      Show " << p << " command-line interface options.\n";
+}
+
+int main(int argc, char **argv){
+    size_t response_maxlen = 500;
+    std::string savefile;
+    std::string load_txt;
+    std::string load_kb;
+    int def_depth = 3;
+    int n_gram_size = 3;
+    double repeat_penalty = 0.7; // default λ
+    std::vector<std::string> learn_files;
+    bool learn_chat = true;
+    bool allow_path_loops = false;
+
+    for (int i=1;i<argc;++i){
+        std::string a = argv[i];
+        if (a=="--help"){ print_commands(argv[0]); return 0; }
+        if (a=="--response-max-length" && i+1<argc){ response_maxlen = std::stoul(argv[++i]); continue; }
+        if (a=="--save-kb" && i+1<argc){ savefile = argv[++i]; continue; }
+        if (a=="--load-kb" && i+1<argc){ load_kb = argv[++i]; continue; }
+        if (a=="--info-kb" && i+1<argc){ print_kb_info(argv[++i]); return 0; }
+        if (a=="--static-kb"){ learn_chat = false; continue; }
+        if (a=="--dictionary-depth" && i+1<argc){ def_depth = std::stoi(argv[++i]); continue; }
+        if (a=="--n-gram-max-size" && i+1<argc){ n_gram_size = std::max(1, std::stoi(argv[++i])); continue; }
+        if (a=="--learn"){ while(i+1<argc && argv[i+1][0] != '-') learn_files.push_back(argv[++i]); continue; }
+        if (a=="--repeat-penalty" && i+1<argc){ repeat_penalty = std::stod(argv[++i]); continue; }
+        if (a=="--allow-path-loops") { allow_path_loops = true; continue; }
+        if (a=="--activate-agi"){
+                std::cerr << "AGI subscription fee is $1,000 CAD per month. To pay the fee, contact Caleb Nwokocha via email cnwokocha@proton.me\n";
+                return 0;
+        }
+
+        std::cerr << "Error: Unknown or incomplete command-line option '" << a << "'\n";
+        print_commands(argv[0]);
+        return 1;
+    }
+
+    KnowledgeBase kb;
+
+    global_dictionary_entries = parse_dictionary_json();
+    build_def_tokens_cache();
+    kb.set_def_depth(def_depth);
+
+    if (!load_kb.empty()){
+        try { std::cerr << "Loading knowledge base: " << load_kb << "\n";
+        load_kb_binary(kb, load_kb, def_depth); std::cerr << "Loaded knowledge base: " << load_kb << "\n"; }
+        catch (const std::exception &e){ std::cerr << "Error: " << e.what() << "\n"; }
+    }
+
+    if (!learn_files.empty()){
+        std::cerr << "Learning from file/s (" << learn_files.size() << ") using threads=" << omp_get_max_threads() << "\n";
+        learn_files_parallel(kb, learn_files, n_gram_size);
+    }
+
+    std::string line;
+    std::cout << "Ready. Enter prompts.\n";
+    while (std::cout << "> " , std::getline(std::cin, line)){
+        if (line.empty()){ std::cout << "\n"; continue; }
+        auto prompt_toks = tokenize_whitespace(line);
+        if (learn_chat) {learn_tokens_ngram(kb, prompt_toks, n_gram_size);}
+        auto resp = construct_response(kb, prompt_toks, response_maxlen, repeat_penalty, n_gram_size, allow_path_loops);
+        std::cout << "\n";
+        if (!resp.empty() && learn_chat){learn_tokens_ngram(kb, resp, n_gram_size);}
+        if (!savefile.empty()){
+            try { std::cerr << "Saving knowledge base: " << savefile << "\n";
+            save_kb_binary(kb, savefile); std::cerr << "Saved knowledge base: " << savefile << "\n"; }
+            catch (const std::exception &e){ std::cerr << "Error: " << e.what() << "\n"; }
+        }
+    }
+
+    return 0;
+}