Upload 2 files

ChatIPC.cpp

@@ -308,7 +308,7 @@ static std::vector<std::string> tokenize_whitespace(const std::string &s){
     return out;
 }
 
-static std::vector<std::string> tokenize_dictionary_expansion(const std::string &s) {
+static std::vector<std::string> tokenize_other(const std::string &s) {
     std::vector<std::string> out;
     std::string cur;
 
@@ -445,7 +445,7 @@ static void build_def_tokens_cache(){
 
         auto &defs = global_def_tokens_cache[key];
         for (const auto &def : entry.definitions){
-            auto toks = tokenize_dictionary_expansion(def);
+            auto toks = tokenize_other(def);
             defs.insert(defs.end(), toks.begin(), toks.end());
         }
     }
@@ -769,6 +769,9 @@ static std::string best_candidate_by_similarity(
     std::vector<std::uint64_t> agg_words(words, 0ULL);
 
     auto add_token_and_defs = [&](StrPtr t){
+        if (!t) return;
+
+        // 1. Process the main token and its direct definitions
         TokenId tid = interner.id_of(t);
         if (tid != TOKEN_ID_INVALID) bitset_set(agg_words.data(), words, tid);
 
@@ -779,6 +782,32 @@ static std::string best_candidate_by_similarity(
                 if (did != TOKEN_ID_INVALID) bitset_set(agg_words.data(), words, did);
             }
         }
+
+        // 2. Tokenize the expanded dictionary string to process sub-tokens
+        std::vector<std::string> sub_tokens = tokenize_other(*t);
+        for (const std::string& sub_tok : sub_tokens) {
+
+            // Use id_of to check if the sub-token exists without adding a new string to the pool
+            TokenId sub_tid = interner.id_of(sub_tok);
+
+            if (sub_tid != TOKEN_ID_INVALID) {
+                // Add the sub-token itself to the bitset
+                bitset_set(agg_words.data(), words, sub_tid);
+
+                // Retrieve the interned pointer so we can look it up in def_index
+                StrPtr sub_ptr = interner.ptr_from_id(sub_tid);
+                if (sub_ptr) {
+                    auto sub_it = def_index.find(sub_ptr);
+                    if (sub_it != def_index.end()) {
+                        // Add the sub-token's definitions to the bitset
+                        for (StrPtr d : sub_it->second) {
+                            TokenId did = interner.id_of(d);
+                            if (did != TOKEN_ID_INVALID) bitset_set(agg_words.data(), words, did);
+                        }
+                    }
+                }
+            }
+        }
     };
 
     for (StrPtr t : prompt_ptrs) add_token_and_defs(t);
@@ -794,9 +823,11 @@ static std::string best_candidate_by_similarity(
         std::uint64_t *row = cand_words.data() + i * words;
         const StrPtr cand = cands[i];
 
+        // 1. Tag the candidate word itself
         TokenId cid = interner.id_of(cand);
         if (cid != TOKEN_ID_INVALID) bitset_set(row, words, cid);
 
+        // 2. Expand the candidate's direct definition
         auto it = def_index.find(cand);
         if (it != def_index.end()){
             for (StrPtr d : it->second){
@@ -805,6 +836,33 @@ static std::string best_candidate_by_similarity(
             }
         }
 
+        // 3. NEW: Tokenize the candidate to process sub-tokens (e.g., camelCase, hyphens)
+        std::vector<std::string> sub_tokens = tokenize_other(*cand);
+        for (const std::string& sub_tok : sub_tokens) {
+
+            // Check if the sub-token exists in our knowledge base
+            TokenId sub_tid = interner.id_of(sub_tok);
+
+            if (sub_tid != TOKEN_ID_INVALID) {
+                // Add the sub-token itself to the candidate's semantic fingerprint (row)
+                bitset_set(row, words, sub_tid);
+
+                // Convert the ID back to a pointer so we can check the dictionary
+                StrPtr sub_ptr = interner.ptr_from_id(sub_tid);
+                if (sub_ptr) {
+                    auto sub_it = def_index.find(sub_ptr);
+                    if (sub_it != def_index.end()) {
+                        // Add the sub-token's deeper definitions to the candidate's row
+                        for (StrPtr d : sub_it->second) {
+                            TokenId did = interner.id_of(d);
+                            if (did != TOKEN_ID_INVALID) bitset_set(row, words, did);
+                        }
+                    }
+                }
+            }
+        }
+
+        // 4. Finally, count the bits for the Jaccard similarity math
         cand_counts[i] = bitset_count(row, words);
     }