Upload ChatIPC.cpp

ChatIPC.cpp

@@ -308,7 +308,7 @@ static std::vector<std::string> tokenize_whitespace(const std::string &s){
     return out;
 }
 
-static std::vector<std::string> tokenize_other(const std::string &s) {
+static std::vector<std::string> tokenize_others(const std::string &s) {
     std::vector<std::string> out;
     std::string cur;
 
@@ -445,7 +445,7 @@ static void build_def_tokens_cache(){
 
         auto &defs = global_def_tokens_cache[key];
         for (const auto &def : entry.definitions){
-            auto toks = tokenize_other(def);
+            auto toks = tokenize_others(def);
             defs.insert(defs.end(), toks.begin(), toks.end());
         }
     }
@@ -771,7 +771,7 @@ static std::string best_candidate_by_similarity(
     auto add_token_and_defs = [&](StrPtr t){
         if (!t) return;
 
-        // 1. Process the main token and its direct definitions
+        // Process the main token and its direct definitions
         TokenId tid = interner.id_of(t);
         if (tid != TOKEN_ID_INVALID) bitset_set(agg_words.data(), words, tid);
 
@@ -783,8 +783,8 @@ static std::string best_candidate_by_similarity(
             }
         }
 
-        // 2. Tokenize the expanded dictionary string to process sub-tokens
-        std::vector<std::string> sub_tokens = tokenize_other(*t);
+        // Tokenize the expanded dictionary string to process sub-tokens
+        std::vector<std::string> sub_tokens = tokenize_others(*t);
         for (const std::string& sub_tok : sub_tokens) {
 
             // Use id_of to check if the sub-token exists without adding a new string to the pool
@@ -823,11 +823,11 @@ static std::string best_candidate_by_similarity(
         std::uint64_t *row = cand_words.data() + i * words;
         const StrPtr cand = cands[i];
 
-        // 1. Tag the candidate word itself
+        // Tag the candidate word itself
         TokenId cid = interner.id_of(cand);
         if (cid != TOKEN_ID_INVALID) bitset_set(row, words, cid);
 
-        // 2. Expand the candidate's direct definition
+        // Expand the candidate's direct definition
         auto it = def_index.find(cand);
         if (it != def_index.end()){
             for (StrPtr d : it->second){
@@ -836,8 +836,8 @@ static std::string best_candidate_by_similarity(
             }
         }
 
-        // 3. NEW: Tokenize the candidate to process sub-tokens (e.g., camelCase, hyphens)
-        std::vector<std::string> sub_tokens = tokenize_other(*cand);
+        // Tokenize the candidate to process sub-tokens (e.g., camelCase, hyphens)
+        std::vector<std::string> sub_tokens = tokenize_others(*cand);
         for (const std::string& sub_tok : sub_tokens) {
 
             // Check if the sub-token exists in our knowledge base
@@ -862,7 +862,7 @@ static std::string best_candidate_by_similarity(
             }
         }
 
-        // 4. Finally, count the bits for the Jaccard similarity math
+        // Finally, count the bits for the Jaccard similarity math
         cand_counts[i] = bitset_count(row, words);
     }