Upload 2 files

ChatIPC.cpp

@@ -16,6 +16,7 @@
 #include <unordered_map>
 #include <unordered_set>
 #include <vector>
+#include <string>
 
 #ifdef _OPENMP
 #include <omp.h>
@@ -308,13 +309,108 @@ static std::vector<std::string> tokenize_whitespace(const std::string &s){
     return out;
 }
 
+// Static helper to check dictionary presence using your existing pos cache
+static inline bool check_dictionary(const std::string &word) {
+    return !dictionary_pos_for_token(word).empty();
+}
+
+// Morphological lemmatizer to convert sub-tokens to their dictionary-defined base form.
+// Returns std::optional<std::string>: contains the valid base form if found in the dictionary,
+// or std::nullopt if the token has no definition and should be ignored.
+static std::optional<std::string> get_valid_base_form(const std::string &word) {
+    if (word.empty()) return std::nullopt;
+
+    // 1. BEFORE extracting/stemming, check if the original form is already in the dictionary
+    if (check_dictionary(word)) {
+        return word;
+    }
+
+    // 2. Continuous verb form (-ing)
+    if (word.length() > 3 && word.substr(word.length() - 3) == "ing") {
+        // e.g., playing -> play
+        std::string base1 = word.substr(0, word.length() - 3);
+        if (check_dictionary(base1)) return base1;
+
+        // e.g., making -> make
+        std::string base2 = base1 + "e";
+        if (check_dictionary(base2)) return base2;
+
+        // e.g., running -> run (doubled consonant check)
+        if (base1.length() >= 2 && base1.back() == base1[base1.length() - 2]) {
+            std::string base3 = base1.substr(0, base1.length() - 1);
+            if (check_dictionary(base3)) return base3;
+        }
+    }
+
+    // 3. Past tense / Participle form (-ed)
+    if (word.length() > 2 && word.substr(word.length() - 2) == "ed") {
+        // e.g., baked -> bake
+        std::string base1 = word.substr(0, word.length() - 1);
+        if (check_dictionary(base1)) return base1;
+
+        // e.g., played -> play
+        std::string base2 = word.substr(0, word.length() - 2);
+        if (check_dictionary(base2)) return base2;
+
+        // e.g., hopped -> hop (doubled consonant check)
+        if (base2.length() >= 2 && base2.back() == base2[base2.length() - 2]) {
+            std::string base3 = base2.substr(0, base2.length() - 1);
+            if (check_dictionary(base3)) return base3;
+        }
+    }
+
+    // 4. Plural / 3rd-person singular form (-ies -> -y)
+    if (word.length() > 3 && word.substr(word.length() - 3) == "ies") {
+        std::string base = word.substr(0, word.length() - 3) + "y";
+        if (check_dictionary(base)) return base;
+    }
+
+    // 5. Plural / 3rd-person singular form (-ves -> -f / -fe)
+    if (word.length() > 3 && word.substr(word.length() - 3) == "ves") {
+        std::string base1 = word.substr(0, word.length() - 3) + "f";
+        if (check_dictionary(base1)) return base1;
+
+        std::string base2 = word.substr(0, word.length() - 3) + "fe";
+        if (check_dictionary(base2)) return base2;
+    }
+
+    // 6. Plural form (-es)
+    if (word.length() > 2 && word.substr(word.length() - 2) == "es") {
+        std::string base = word.substr(0, word.length() - 2);
+        if (check_dictionary(base)) return base;
+    }
+
+    // 7. Plural form (-s)
+    if (word.length() > 1 && word.back() == 's' && word[word.length() - 2] != 's') {
+        std::string base = word.substr(0, word.length() - 1);
+        if (check_dictionary(base)) return base;
+    }
+
+    // No dictionary-defined base form found
+    return std::nullopt;
+}
+
 static std::vector<std::string> tokenize_others(const std::string &s) {
+    // Step A: Check if the entire input string already has a dictionary definition.
+    if (check_dictionary(s)) {
+        std::string lower_s;
+        lower_s.reserve(s.size());
+        for (char c : s) {
+            lower_s.push_back(to_low(c));
+        }
+        return { lower_s };
+    }
+
     std::vector<std::string> out;
     std::string cur;
 
+    // Lambda to flush the accumulated token buffer, validate it, and add to output
     auto flush = [&]() {
         if (!cur.empty()) {
-            out.push_back(cur);
+            auto valid_base = get_valid_base_form(cur);
+            if (valid_base.has_value()) {
+                out.push_back(valid_base.value());
+            }
             cur.clear();
         }
     };
@@ -323,6 +419,7 @@ static std::vector<std::string> tokenize_others(const std::string &s) {
         unsigned char uc = static_cast<unsigned char>(s[i]);
         char ch = static_cast<char>(uc);
 
+        // Split on spaces, underscores, hyphens, and slashes
         if (ch == '_' || ch == '-' || ch == '/' || std::isspace(uc)) {
             flush();
             continue;
@@ -341,7 +438,18 @@ static std::vector<std::string> tokenize_others(const std::string &s) {
             camel_boundary = prev_lower_or_digit || (prev_upper && next_lower);
         }
 
-        if (camel_boundary) {
+        // Split on transitions between alphabetical characters and digits (e.g., "book5read")
+        bool digit_boundary = false;
+        if (!cur.empty()) {
+            unsigned char prev = static_cast<unsigned char>(s[i - 1]);
+            bool prev_digit = std::isdigit(prev) != 0;
+            bool curr_digit = std::isdigit(uc) != 0;
+            if (prev_digit != curr_digit) {
+                digit_boundary = true;
+            }
+        }
+
+        if (camel_boundary || digit_boundary) {
             flush();
         }