Upload ChatIPC.cpp
Browse files- ChatIPC.cpp +415 -421
ChatIPC.cpp
CHANGED
|
@@ -1,18 +1,15 @@
|
|
| 1 |
#include <algorithm>
|
| 2 |
-
#include <atomic>
|
| 3 |
#include <cctype>
|
| 4 |
-
#include <
|
| 5 |
-
#include <
|
| 6 |
#include <fstream>
|
| 7 |
#include <iostream>
|
| 8 |
#include <iterator>
|
| 9 |
-
#include <map>
|
| 10 |
#include <mutex>
|
| 11 |
#include <optional>
|
| 12 |
#include <sstream>
|
| 13 |
#include <stdexcept>
|
| 14 |
#include <string>
|
| 15 |
-
#include <thread>
|
| 16 |
#include <unordered_map>
|
| 17 |
#include <unordered_set>
|
| 18 |
#include <vector>
|
|
@@ -21,49 +18,48 @@
|
|
| 21 |
#include <omp.h>
|
| 22 |
#else
|
| 23 |
inline int omp_get_max_threads(){ return 1; }
|
| 24 |
-
inline int omp_get_thread_num(){ return 0; }
|
| 25 |
#endif
|
| 26 |
|
| 27 |
extern unsigned char dictionary_json[];
|
| 28 |
extern unsigned int dictionary_json_len;
|
| 29 |
|
|
|
|
| 30 |
|
| 31 |
-
static inline bool is_space(char c){ return
|
| 32 |
-
static inline char to_low(char c){ return static_cast<char>(
|
| 33 |
-
static inline void safe_flush(std::ostream &os){ os.flush(); }
|
| 34 |
|
| 35 |
struct DictionaryEntry {
|
| 36 |
-
|
| 37 |
-
|
| 38 |
-
|
| 39 |
};
|
| 40 |
|
| 41 |
-
static
|
| 42 |
-
static
|
| 43 |
-
static
|
| 44 |
|
| 45 |
static inline bool is_word_char_for_key(char c){
|
| 46 |
unsigned char uc = static_cast<unsigned char>(c);
|
| 47 |
-
return
|
| 48 |
}
|
| 49 |
|
| 50 |
-
static
|
| 51 |
size_t b = 0, e = s.size();
|
| 52 |
while (b < e && !is_word_char_for_key(s[b])) ++b;
|
| 53 |
while (e > b && !is_word_char_for_key(s[e - 1])) --e;
|
| 54 |
|
| 55 |
-
|
| 56 |
out.reserve(e - b);
|
| 57 |
for (size_t i = b; i < e; ++i) out.push_back(to_low(s[i]));
|
| 58 |
return out;
|
| 59 |
}
|
| 60 |
|
| 61 |
-
static
|
| 62 |
-
|
| 63 |
out.reserve(s.size());
|
| 64 |
for (char c : s){
|
| 65 |
unsigned char uc = static_cast<unsigned char>(c);
|
| 66 |
-
if (
|
| 67 |
}
|
| 68 |
return out;
|
| 69 |
}
|
|
@@ -82,7 +78,7 @@ enum class PosClass {
|
|
| 82 |
Interj
|
| 83 |
};
|
| 84 |
|
| 85 |
-
static PosClass pos_class_from_tag(const
|
| 86 |
if (tag == "n" || tag == "noun") return PosClass::Noun;
|
| 87 |
if (tag == "v" || tag == "verb" || tag == "part" || tag == "participle" || tag == "p") return PosClass::Verb;
|
| 88 |
if (tag == "a" || tag == "adj" || tag == "adjective") return PosClass::Adj;
|
|
@@ -96,15 +92,15 @@ static PosClass pos_class_from_tag(const std::string &tag){
|
|
| 96 |
return PosClass::Unknown;
|
| 97 |
}
|
| 98 |
|
| 99 |
-
static bool has_pos_class(const
|
| 100 |
for (const auto &t : tags){
|
| 101 |
if (pos_class_from_tag(t) == cls) return true;
|
| 102 |
}
|
| 103 |
return false;
|
| 104 |
}
|
| 105 |
|
| 106 |
-
static const
|
| 107 |
-
static const
|
| 108 |
auto key = normalize_dictionary_key(surface);
|
| 109 |
if (key.empty()) return empty;
|
| 110 |
|
|
@@ -112,49 +108,49 @@ static const std::vector<std::string> &dictionary_pos_for_token(const std::strin
|
|
| 112 |
return (it == global_pos_cache.end()) ? empty : it->second;
|
| 113 |
}
|
| 114 |
|
| 115 |
-
static bool first_alpha_is_upper(const
|
| 116 |
for (char c : s){
|
| 117 |
unsigned char uc = static_cast<unsigned char>(c);
|
| 118 |
-
if (
|
| 119 |
}
|
| 120 |
return false;
|
| 121 |
}
|
| 122 |
|
| 123 |
-
static bool first_alpha_is_lower(const
|
| 124 |
for (char c : s){
|
| 125 |
unsigned char uc = static_cast<unsigned char>(c);
|
| 126 |
-
if (
|
| 127 |
}
|
| 128 |
return false;
|
| 129 |
}
|
| 130 |
|
| 131 |
-
static bool is_sentence_boundary_token(const
|
| 132 |
if (s.empty()) return false;
|
| 133 |
char c = s.back();
|
| 134 |
return c == '.' || c == '!' || c == '?';
|
| 135 |
}
|
| 136 |
|
| 137 |
-
static bool is_open_punct_token(const
|
| 138 |
return s == "(" || s == "[" || s == "{" || s == "\"" || s == "'";
|
| 139 |
}
|
| 140 |
|
| 141 |
-
static bool is_punctuation_only_token(const
|
| 142 |
if (s.empty()) return false;
|
| 143 |
for (char c : s){
|
| 144 |
unsigned char uc = static_cast<unsigned char>(c);
|
| 145 |
-
if (
|
| 146 |
}
|
| 147 |
return true;
|
| 148 |
}
|
| 149 |
|
| 150 |
-
static
|
| 151 |
-
|
| 152 |
-
[](unsigned char c) { return static_cast<char>(
|
| 153 |
return s;
|
| 154 |
}
|
| 155 |
|
| 156 |
-
static bool is_common_determiner(const
|
| 157 |
-
static const
|
| 158 |
"a", "an", "the", "this", "that", "these", "those",
|
| 159 |
"my", "your", "his", "her", "its", "our", "their",
|
| 160 |
"all", "some", "any", "each", "every", "either", "neither",
|
|
@@ -167,8 +163,8 @@ static bool is_common_determiner(const std::string &s) {
|
|
| 167 |
return words.find(to_lower_ascii(s)) != words.end();
|
| 168 |
}
|
| 169 |
|
| 170 |
-
static bool is_common_preposition(const
|
| 171 |
-
static const
|
| 172 |
"of", "in", "on", "at", "by", "for", "from", "with",
|
| 173 |
"into", "onto", "about", "over", "under", "after", "before",
|
| 174 |
"between", "through", "during", "without", "within", "across",
|
|
@@ -184,8 +180,8 @@ static bool is_common_preposition(const std::string &s) {
|
|
| 184 |
return words.find(to_lower_ascii(s)) != words.end();
|
| 185 |
}
|
| 186 |
|
| 187 |
-
static bool is_common_aux_or_modal(const
|
| 188 |
-
static const
|
| 189 |
"to", "be", "am", "is", "are", "was", "were", "been", "being",
|
| 190 |
"have", "has", "had", "do", "does", "did",
|
| 191 |
"can", "could", "may", "might", "must", "shall", "should", "will",
|
|
@@ -197,17 +193,17 @@ static bool is_common_aux_or_modal(const std::string &s) {
|
|
| 197 |
return words.find(to_lower_ascii(s)) != words.end();
|
| 198 |
}
|
| 199 |
|
| 200 |
-
static bool begins_with_vowel_sound(const
|
| 201 |
if (s.empty()) return false;
|
| 202 |
|
| 203 |
-
const
|
| 204 |
|
| 205 |
-
static const
|
| 206 |
"hour", "hours", "honest", "honesty", "honor", "honour", "honorable", "honourable",
|
| 207 |
"heir", "heiress", "herb", "herbal", "herbalist", "homage", "hono(u)rary"
|
| 208 |
};
|
| 209 |
|
| 210 |
-
static const
|
| 211 |
"uni", "univ", "unit", "use", "user", "euro", "eu", "one", "once", "ouija",
|
| 212 |
"ufo", "ut", "uk", "ubiq", "ewe", "eul", "eup", "x", "y"
|
| 213 |
};
|
|
@@ -224,14 +220,13 @@ static bool begins_with_vowel_sound(const std::string &s){
|
|
| 224 |
}
|
| 225 |
}
|
| 226 |
|
| 227 |
-
if (t.empty()) return false;
|
| 228 |
char c = t[0];
|
| 229 |
return c == 'a' || c == 'e' || c == 'i' || c == 'o' || c == 'u';
|
| 230 |
}
|
| 231 |
|
| 232 |
-
static double english_rule_bonus(const
|
| 233 |
-
const
|
| 234 |
-
const
|
| 235 |
|
| 236 |
const auto &ctx_tags = dictionary_pos_for_token(context_tok);
|
| 237 |
const auto &cand_tags = dictionary_pos_for_token(cand);
|
|
@@ -292,7 +287,7 @@ static double english_rule_bonus(const std::string &context_tok, const std::stri
|
|
| 292 |
|
| 293 |
if (is_punctuation_only_token(cand)){
|
| 294 |
if (sentence_start) bonus -= 0.05;
|
| 295 |
-
else if (!context_tok.empty() &&
|
| 296 |
}
|
| 297 |
|
| 298 |
if (is_sentence_boundary_token(cand)) bonus += 0.06;
|
|
@@ -300,24 +295,24 @@ static double english_rule_bonus(const std::string &context_tok, const std::stri
|
|
| 300 |
return bonus;
|
| 301 |
}
|
| 302 |
|
| 303 |
-
static
|
| 304 |
-
|
| 305 |
-
|
| 306 |
-
|
| 307 |
while (iss >> t) out.push_back(t);
|
| 308 |
return out;
|
| 309 |
}
|
| 310 |
|
| 311 |
// Static helper to check dictionary presence using the existing pos cache
|
| 312 |
-
static inline bool check_dictionary(const
|
| 313 |
return !dictionary_pos_for_token(word).empty();
|
| 314 |
}
|
| 315 |
|
| 316 |
// Morphological lemmatizer to convert sub-tokens to their dictionary-defined base form.
|
| 317 |
-
// Returns
|
| 318 |
-
// or
|
| 319 |
-
static
|
| 320 |
-
if (word.empty()) return
|
| 321 |
|
| 322 |
// 1. BEFORE extracting/stemming, check if the original form is already in the dictionary
|
| 323 |
if (check_dictionary(word)) {
|
|
@@ -327,16 +322,16 @@ static std::optional<std::string> get_valid_base_form(const std::string &word) {
|
|
| 327 |
// 2. Continuous verb form (-ing)
|
| 328 |
if (word.length() > 3 && word.substr(word.length() - 3) == "ing") {
|
| 329 |
// e.g., playing -> play
|
| 330 |
-
|
| 331 |
if (check_dictionary(base1)) return base1;
|
| 332 |
|
| 333 |
// e.g., making -> make
|
| 334 |
-
|
| 335 |
if (check_dictionary(base2)) return base2;
|
| 336 |
|
| 337 |
// e.g., running -> run (doubled consonant check)
|
| 338 |
if (base1.length() >= 2 && base1.back() == base1[base1.length() - 2]) {
|
| 339 |
-
|
| 340 |
if (check_dictionary(base3)) return base3;
|
| 341 |
}
|
| 342 |
}
|
|
@@ -344,16 +339,16 @@ static std::optional<std::string> get_valid_base_form(const std::string &word) {
|
|
| 344 |
// 3. Past tense / Participle form (-ed)
|
| 345 |
if (word.length() > 2 && word.substr(word.length() - 2) == "ed") {
|
| 346 |
// e.g., baked -> bake
|
| 347 |
-
|
| 348 |
if (check_dictionary(base1)) return base1;
|
| 349 |
|
| 350 |
// e.g., played -> play
|
| 351 |
-
|
| 352 |
if (check_dictionary(base2)) return base2;
|
| 353 |
|
| 354 |
// e.g., hopped -> hop (doubled consonant check)
|
| 355 |
if (base2.length() >= 2 && base2.back() == base2[base2.length() - 2]) {
|
| 356 |
-
|
| 357 |
if (check_dictionary(base3)) return base3;
|
| 358 |
}
|
| 359 |
}
|
|
@@ -361,22 +356,22 @@ static std::optional<std::string> get_valid_base_form(const std::string &word) {
|
|
| 361 |
// 4. Superlative form (-est)
|
| 362 |
if (word.length() > 3 && word.substr(word.length() - 3) == "est") {
|
| 363 |
// e.g., simplest -> simple (strip 'st')
|
| 364 |
-
|
| 365 |
if (check_dictionary(base1)) return base1;
|
| 366 |
|
| 367 |
// e.g., fastest -> fast
|
| 368 |
-
|
| 369 |
if (check_dictionary(base2)) return base2;
|
| 370 |
|
| 371 |
// e.g., happiest -> happy (-iest -> -y)
|
| 372 |
if (word.length() > 4 && word.substr(word.length() - 4) == "iest") {
|
| 373 |
-
|
| 374 |
if (check_dictionary(base_y)) return base_y;
|
| 375 |
}
|
| 376 |
|
| 377 |
// e.g., biggest -> big (doubled consonant check)
|
| 378 |
if (base2.length() >= 2 && base2.back() == base2[base2.length() - 2]) {
|
| 379 |
-
|
| 380 |
if (check_dictionary(base3)) return base3;
|
| 381 |
}
|
| 382 |
}
|
|
@@ -384,22 +379,22 @@ static std::optional<std::string> get_valid_base_form(const std::string &word) {
|
|
| 384 |
// 5. Comparative form (-er)
|
| 385 |
if (word.length() > 2 && word.substr(word.length() - 2) == "er") {
|
| 386 |
// e.g., simpler -> simple (strip 'r')
|
| 387 |
-
|
| 388 |
if (check_dictionary(base1)) return base1;
|
| 389 |
|
| 390 |
// e.g., faster -> fast
|
| 391 |
-
|
| 392 |
if (check_dictionary(base2)) return base2;
|
| 393 |
|
| 394 |
// e.g., happier -> happy (-ier -> -y)
|
| 395 |
if (word.length() > 3 && word.substr(word.length() - 3) == "ier") {
|
| 396 |
-
|
| 397 |
if (check_dictionary(base_y)) return base_y;
|
| 398 |
}
|
| 399 |
|
| 400 |
// e.g., bigger -> big (doubled consonant check)
|
| 401 |
if (base2.length() >= 2 && base2.back() == base2[base2.length() - 2]) {
|
| 402 |
-
|
| 403 |
if (check_dictionary(base3)) return base3;
|
| 404 |
}
|
| 405 |
}
|
|
@@ -407,27 +402,27 @@ static std::optional<std::string> get_valid_base_form(const std::string &word) {
|
|
| 407 |
// 6. Adverbial form (-ly)
|
| 408 |
if (word.length() > 2 && word.substr(word.length() - 2) == "ly") {
|
| 409 |
// e.g., quickly -> quick
|
| 410 |
-
|
| 411 |
if (check_dictionary(base1)) return base1;
|
| 412 |
|
| 413 |
// e.g., happily -> happy (-ily -> -y)
|
| 414 |
if (word.length() > 3 && word.substr(word.length() - 3) == "ily") {
|
| 415 |
-
|
| 416 |
if (check_dictionary(base2)) return base2;
|
| 417 |
}
|
| 418 |
|
| 419 |
// e.g., basically -> basic (-ally -> -ic)
|
| 420 |
if (word.length() > 4 && word.substr(word.length() - 4) == "ally") {
|
| 421 |
-
|
| 422 |
if (check_dictionary(base3)) return base3;
|
| 423 |
|
| 424 |
-
|
| 425 |
if (check_dictionary(base4)) return base4;
|
| 426 |
}
|
| 427 |
|
| 428 |
// e.g., gently -> gentle (-ly -> -le)
|
| 429 |
if (word.back() == 'y' && word.length() > 2 && word[word.length() - 2] == 'l') {
|
| 430 |
-
|
| 431 |
if (check_dictionary(base5)) return base5;
|
| 432 |
}
|
| 433 |
}
|
|
@@ -435,12 +430,12 @@ static std::optional<std::string> get_valid_base_form(const std::string &word) {
|
|
| 435 |
// 7. Noun suffix (-ness)
|
| 436 |
if (word.length() > 4 && word.substr(word.length() - 4) == "ness") {
|
| 437 |
// e.g., sadness -> sad
|
| 438 |
-
|
| 439 |
if (check_dictionary(base1)) return base1;
|
| 440 |
|
| 441 |
// e.g., happiness -> happy (-iness -> -y)
|
| 442 |
if (base1.length() > 1 && base1.back() == 'i') {
|
| 443 |
-
|
| 444 |
if (check_dictionary(base2)) return base2;
|
| 445 |
}
|
| 446 |
}
|
|
@@ -448,39 +443,39 @@ static std::optional<std::string> get_valid_base_form(const std::string &word) {
|
|
| 448 |
// 8. Noun suffix (-ment)
|
| 449 |
if (word.length() > 4 && word.substr(word.length() - 4) == "ment") {
|
| 450 |
// e.g., development -> develop
|
| 451 |
-
|
| 452 |
if (check_dictionary(base1)) return base1;
|
| 453 |
}
|
| 454 |
|
| 455 |
// 9. Plural / 3rd-person singular form (-ies -> -y)
|
| 456 |
if (word.length() > 3 && word.substr(word.length() - 3) == "ies") {
|
| 457 |
-
|
| 458 |
if (check_dictionary(base)) return base;
|
| 459 |
}
|
| 460 |
|
| 461 |
// 10. Plural / 3rd-person singular form (-ves -> -f / -fe)
|
| 462 |
if (word.length() > 3 && word.substr(word.length() - 3) == "ves") {
|
| 463 |
-
|
| 464 |
if (check_dictionary(base1)) return base1;
|
| 465 |
|
| 466 |
-
|
| 467 |
if (check_dictionary(base2)) return base2;
|
| 468 |
}
|
| 469 |
|
| 470 |
// 11. Plural form (-es)
|
| 471 |
if (word.length() > 2 && word.substr(word.length() - 2) == "es") {
|
| 472 |
-
|
| 473 |
if (check_dictionary(base)) return base;
|
| 474 |
}
|
| 475 |
|
| 476 |
// 12. Plural form (-s)
|
| 477 |
if (word.length() > 1 && word.back() == 's' && word[word.length() - 2] != 's') {
|
| 478 |
-
|
| 479 |
if (check_dictionary(base)) return base;
|
| 480 |
}
|
| 481 |
|
| 482 |
// No dictionary-defined base form found
|
| 483 |
-
return
|
| 484 |
}
|
| 485 |
|
| 486 |
// Helper arrays for number-to-words
|
|
@@ -494,23 +489,23 @@ static const char* const scales[] = {"", "thousand", "million", "billion", "tril
|
|
| 494 |
"quattuordecillion", "quindecillion", "sexdecillion", "septendecillion",
|
| 495 |
"octodecillion", "novemdecillion", "vigintillion", "googol"};
|
| 496 |
|
| 497 |
-
static
|
| 498 |
-
|
| 499 |
if (input_s.empty()) return words;
|
| 500 |
|
| 501 |
// Split the number into integer and fractional parts
|
| 502 |
-
|
| 503 |
-
|
| 504 |
size_t dot_pos = input_s.find('.');
|
| 505 |
-
if (dot_pos !=
|
| 506 |
int_part = input_s.substr(0, dot_pos);
|
| 507 |
frac_part = input_s.substr(dot_pos + 1);
|
| 508 |
}
|
| 509 |
|
| 510 |
// Clean the integer part by removing commas
|
| 511 |
-
|
| 512 |
for (char c : int_part) {
|
| 513 |
-
if (
|
| 514 |
s.push_back(c);
|
| 515 |
}
|
| 516 |
}
|
|
@@ -520,16 +515,16 @@ static std::vector<std::string> digit_to_words(const std::string& input_s) {
|
|
| 520 |
// If there are no integer digits, ensure there are fractional digits before parsing
|
| 521 |
bool has_frac_digits = false;
|
| 522 |
for (char c : frac_part) {
|
| 523 |
-
if (
|
| 524 |
}
|
| 525 |
if (!has_frac_digits) return words;
|
| 526 |
words.push_back("zero");
|
| 527 |
} else {
|
| 528 |
size_t first_non_zero = s.find_first_not_of('0');
|
| 529 |
-
if (first_non_zero ==
|
| 530 |
words.push_back("zero");
|
| 531 |
} else {
|
| 532 |
-
|
| 533 |
|
| 534 |
// Fallback for extremely large numbers to prevent overflow
|
| 535 |
if (num.length() > 69) {
|
|
@@ -539,13 +534,13 @@ static std::vector<std::string> digit_to_words(const std::string& input_s) {
|
|
| 539 |
} else {
|
| 540 |
int scale_idx = 0;
|
| 541 |
while (!num.empty()) {
|
| 542 |
-
int chunk_len =
|
| 543 |
-
|
| 544 |
num = num.substr(0, num.length() - chunk_len);
|
| 545 |
|
| 546 |
-
int chunk =
|
| 547 |
if (chunk > 0) {
|
| 548 |
-
|
| 549 |
int h = chunk / 100;
|
| 550 |
int remainder = chunk % 100;
|
| 551 |
|
|
@@ -581,12 +576,12 @@ static std::vector<std::string> digit_to_words(const std::string& input_s) {
|
|
| 581 |
if (!frac_part.empty()) {
|
| 582 |
bool has_frac_digits = false;
|
| 583 |
for (char c : frac_part) {
|
| 584 |
-
if (
|
| 585 |
}
|
| 586 |
if (has_frac_digits) {
|
| 587 |
words.push_back("point");
|
| 588 |
for (char c : frac_part) {
|
| 589 |
-
if (
|
| 590 |
words.push_back(ones[c - '0']);
|
| 591 |
}
|
| 592 |
}
|
|
@@ -596,9 +591,9 @@ static std::vector<std::string> digit_to_words(const std::string& input_s) {
|
|
| 596 |
return words;
|
| 597 |
}
|
| 598 |
|
| 599 |
-
static
|
| 600 |
if (check_dictionary(s)) {
|
| 601 |
-
|
| 602 |
lower_s.reserve(s.size());
|
| 603 |
for (char c : s) {
|
| 604 |
lower_s.push_back(to_low(c));
|
|
@@ -608,26 +603,26 @@ static std::vector<std::string> tokenize_others(const std::string &s) {
|
|
| 608 |
|
| 609 |
// Check if the entire string is a formatted number (pure digits, commas, and dots)
|
| 610 |
auto is_num_char = [](unsigned char c) {
|
| 611 |
-
return
|
| 612 |
};
|
| 613 |
-
bool is_all_digits = !s.empty() &&
|
| 614 |
-
|
| 615 |
|
| 616 |
if (is_all_digits) {
|
| 617 |
return digit_to_words(s);
|
| 618 |
}
|
| 619 |
|
| 620 |
-
|
| 621 |
-
|
| 622 |
|
| 623 |
auto flush = [&]() {
|
| 624 |
if (!cur.empty()) {
|
| 625 |
// Check numbers inside mixed tokens (e.g. "word1,234.56")
|
| 626 |
-
bool cur_all_digits =
|
| 627 |
-
return
|
| 628 |
});
|
| 629 |
-
bool has_digits =
|
| 630 |
-
return
|
| 631 |
});
|
| 632 |
|
| 633 |
if (cur_all_digits && has_digits) {
|
|
@@ -648,25 +643,25 @@ static std::vector<std::string> tokenize_others(const std::string &s) {
|
|
| 648 |
char ch = static_cast<char>(uc);
|
| 649 |
|
| 650 |
// Check for Operator/Symbol extraction
|
| 651 |
-
if (
|
| 652 |
|
| 653 |
// Bypass regular punctuation extraction inside a formatted number
|
| 654 |
if ((ch == ',' || ch == '.') &&
|
| 655 |
!cur.empty() &&
|
| 656 |
-
(
|
| 657 |
-
(i + 1 < s.size()) &&
|
| 658 |
cur.push_back(ch);
|
| 659 |
continue;
|
| 660 |
}
|
| 661 |
|
| 662 |
flush();
|
| 663 |
-
|
| 664 |
|
| 665 |
// Check for double-character operators
|
| 666 |
if (i + 1 < s.size()) {
|
| 667 |
unsigned char nxt_uc = static_cast<unsigned char>(s[i+1]);
|
| 668 |
-
if (
|
| 669 |
-
|
| 670 |
if (check_dictionary(sym2)) {
|
| 671 |
out.push_back(sym2);
|
| 672 |
++i;
|
|
@@ -680,26 +675,26 @@ static std::vector<std::string> tokenize_others(const std::string &s) {
|
|
| 680 |
continue;
|
| 681 |
}
|
| 682 |
|
| 683 |
-
if (
|
| 684 |
flush();
|
| 685 |
continue;
|
| 686 |
}
|
| 687 |
|
| 688 |
-
const bool is_upper =
|
| 689 |
bool camel_boundary = false;
|
| 690 |
if (is_upper && !cur.empty()) {
|
| 691 |
unsigned char prev = static_cast<unsigned char>(s[i - 1]);
|
| 692 |
-
const bool prev_lower_or_digit = (
|
| 693 |
-
const bool prev_upper =
|
| 694 |
-
const bool next_lower = (i + 1 < s.size()) && (
|
| 695 |
camel_boundary = prev_lower_or_digit || (prev_upper && next_lower);
|
| 696 |
}
|
| 697 |
|
| 698 |
bool digit_boundary = false;
|
| 699 |
if (!cur.empty()) {
|
| 700 |
unsigned char prev = static_cast<unsigned char>(s[i - 1]);
|
| 701 |
-
bool prev_digit = (
|
| 702 |
-
bool curr_digit =
|
| 703 |
if (prev_digit != curr_digit) {
|
| 704 |
digit_boundary = true;
|
| 705 |
}
|
|
@@ -709,8 +704,8 @@ static std::vector<std::string> tokenize_others(const std::string &s) {
|
|
| 709 |
flush();
|
| 710 |
}
|
| 711 |
|
| 712 |
-
if (
|
| 713 |
-
cur.push_back(static_cast<char>(
|
| 714 |
} else {
|
| 715 |
flush();
|
| 716 |
}
|
|
@@ -719,26 +714,26 @@ static std::vector<std::string> tokenize_others(const std::string &s) {
|
|
| 719 |
return out;
|
| 720 |
}
|
| 721 |
|
| 722 |
-
using StrPtr = const
|
| 723 |
-
using TokenId =
|
| 724 |
static constexpr TokenId TOKEN_ID_INVALID = 0xFFFFFFFFu;
|
| 725 |
|
| 726 |
-
static inline
|
| 727 |
#if defined(_MSC_VER)
|
| 728 |
-
return static_cast<
|
| 729 |
#else
|
| 730 |
-
return static_cast<
|
| 731 |
#endif
|
| 732 |
}
|
| 733 |
|
| 734 |
struct StringInterner {
|
| 735 |
-
|
| 736 |
-
|
| 737 |
-
|
| 738 |
-
mutable
|
| 739 |
|
| 740 |
-
const
|
| 741 |
-
|
| 742 |
|
| 743 |
auto [it, inserted] = pool.emplace(s);
|
| 744 |
if (inserted){
|
|
@@ -750,45 +745,45 @@ struct StringInterner {
|
|
| 750 |
return &*it;
|
| 751 |
}
|
| 752 |
|
| 753 |
-
TokenId id_of(const
|
| 754 |
-
|
| 755 |
|
| 756 |
auto it = id_by_string.find(s);
|
| 757 |
return (it == id_by_string.end()) ? TOKEN_ID_INVALID : it->second;
|
| 758 |
}
|
| 759 |
|
| 760 |
-
TokenId id_of(const
|
| 761 |
return p ? id_of(*p) : TOKEN_ID_INVALID;
|
| 762 |
}
|
| 763 |
|
| 764 |
-
const
|
| 765 |
-
|
| 766 |
|
| 767 |
return (id < string_by_id.size()) ? string_by_id[(size_t)id] : nullptr;
|
| 768 |
}
|
| 769 |
|
| 770 |
-
|
| 771 |
-
|
| 772 |
return string_by_id.size();
|
| 773 |
}
|
| 774 |
};
|
| 775 |
|
| 776 |
-
static inline void bitset_set(
|
| 777 |
if (id == TOKEN_ID_INVALID) return;
|
| 778 |
-
const
|
| 779 |
if (idx >= words) return;
|
| 780 |
bits[idx] |= (1ULL << (id & 63u));
|
| 781 |
}
|
| 782 |
|
| 783 |
-
static inline
|
| 784 |
-
|
| 785 |
-
for (
|
| 786 |
return total;
|
| 787 |
}
|
| 788 |
|
| 789 |
-
static inline
|
| 790 |
-
|
| 791 |
-
for (
|
| 792 |
return total;
|
| 793 |
}
|
| 794 |
|
|
@@ -800,23 +795,23 @@ static void build_def_tokens_cache(){
|
|
| 800 |
|
| 801 |
#pragma omp parallel
|
| 802 |
{
|
| 803 |
-
|
| 804 |
-
|
| 805 |
|
| 806 |
#pragma omp for schedule(guided) nowait
|
| 807 |
for (int i = 0; i < num_entries; ++i){
|
| 808 |
const auto &entry = global_dictionary_entries[i];
|
| 809 |
|
| 810 |
-
|
| 811 |
for (const auto &def : entry.definitions){
|
| 812 |
auto toks = tokenize_others(def);
|
| 813 |
all_def_toks.insert(all_def_toks.end(), toks.begin(), toks.end());
|
| 814 |
}
|
| 815 |
|
| 816 |
-
|
| 817 |
|
| 818 |
-
|
| 819 |
-
|
| 820 |
for (char c : entry.word) {
|
| 821 |
if (c == ';' || c == ',') {
|
| 822 |
if (!current_sub.empty()) {
|
|
@@ -830,12 +825,12 @@ static void build_def_tokens_cache(){
|
|
| 830 |
if (!current_sub.empty()) sub_words.push_back(current_sub);
|
| 831 |
|
| 832 |
for (size_t j = 0; j < sub_words.size(); ++j) {
|
| 833 |
-
const
|
| 834 |
if (key.empty()) continue;
|
| 835 |
|
| 836 |
if (!pos.empty()) {
|
| 837 |
if (j == sub_words.size() - 1) {
|
| 838 |
-
local_pos_cache[key].push_back(
|
| 839 |
} else {
|
| 840 |
local_pos_cache[key].push_back(pos);
|
| 841 |
}
|
|
@@ -849,49 +844,49 @@ static void build_def_tokens_cache(){
|
|
| 849 |
{
|
| 850 |
for (auto &pr : local_def_cache) {
|
| 851 |
auto &vec = global_def_tokens_cache[pr.first];
|
| 852 |
-
vec.insert(vec.end(),
|
| 853 |
}
|
| 854 |
for (auto &pr : local_pos_cache) {
|
| 855 |
auto &vec = global_pos_cache[pr.first];
|
| 856 |
-
vec.insert(vec.end(),
|
| 857 |
}
|
| 858 |
}
|
| 859 |
}
|
| 860 |
|
| 861 |
// Extract keys to use OpenMP parallel-for deduplication over hash map structures
|
| 862 |
-
|
| 863 |
def_keys.reserve(global_def_tokens_cache.size());
|
| 864 |
for (const auto &pr : global_def_tokens_cache) def_keys.push_back(pr.first);
|
| 865 |
|
| 866 |
#pragma omp parallel for schedule(dynamic)
|
| 867 |
for (ptrdiff_t i = 0; i < static_cast<ptrdiff_t>(def_keys.size()); ++i){
|
| 868 |
auto &v = global_def_tokens_cache[def_keys[i]];
|
| 869 |
-
|
| 870 |
-
v.erase(
|
| 871 |
}
|
| 872 |
|
| 873 |
-
|
| 874 |
pos_keys.reserve(global_pos_cache.size());
|
| 875 |
for (const auto &pr : global_pos_cache) pos_keys.push_back(pr.first);
|
| 876 |
|
| 877 |
#pragma omp parallel for schedule(dynamic)
|
| 878 |
for (ptrdiff_t i = 0; i < static_cast<ptrdiff_t>(pos_keys.size()); ++i){
|
| 879 |
auto &v = global_pos_cache[pos_keys[i]];
|
| 880 |
-
|
| 881 |
-
v.erase(
|
| 882 |
}
|
| 883 |
}
|
| 884 |
|
| 885 |
-
struct PtrHash { size_t operator()(StrPtr p) const noexcept { return
|
| 886 |
struct PtrEq { bool operator()(StrPtr a, StrPtr b) const noexcept { return a == b; } };
|
| 887 |
|
| 888 |
-
using NextSet =
|
| 889 |
|
| 890 |
struct NgramHash {
|
| 891 |
-
|
| 892 |
-
|
| 893 |
for(auto& p : v) {
|
| 894 |
-
seed ^=
|
| 895 |
}
|
| 896 |
return seed;
|
| 897 |
}
|
|
@@ -900,25 +895,25 @@ struct NgramHash {
|
|
| 900 |
struct KnowledgeBase {
|
| 901 |
StringInterner interner;
|
| 902 |
// Modified to use a vector of StrPtr (N-gram) as the key
|
| 903 |
-
|
| 904 |
-
mutable
|
| 905 |
|
| 906 |
-
|
| 907 |
-
mutable
|
| 908 |
int def_depth = 0;
|
| 909 |
|
| 910 |
-
void add_ngram(const
|
| 911 |
for (auto p : ctx) ensure_def_for_interned(p);
|
| 912 |
ensure_def_for_interned(v);
|
| 913 |
|
| 914 |
-
|
| 915 |
auto &vec = next[ctx];
|
| 916 |
for (auto p : vec) if (p == v) return;
|
| 917 |
vec.push_back(v);
|
| 918 |
}
|
| 919 |
|
| 920 |
void set_def_depth(int D){
|
| 921 |
-
|
| 922 |
if (D != def_depth){
|
| 923 |
def_index.clear();
|
| 924 |
def_depth = D;
|
|
@@ -929,14 +924,14 @@ struct KnowledgeBase {
|
|
| 929 |
if (wp == nullptr || def_depth <= 0) return;
|
| 930 |
|
| 931 |
{
|
| 932 |
-
|
| 933 |
if (def_index.find(wp) != def_index.end()) return;
|
| 934 |
}
|
| 935 |
|
| 936 |
-
|
| 937 |
-
|
| 938 |
|
| 939 |
-
const
|
| 940 |
if (!start_key.empty()){
|
| 941 |
auto it_def = global_def_tokens_cache.find(start_key);
|
| 942 |
if (it_def != global_def_tokens_cache.end()){
|
|
@@ -948,10 +943,10 @@ struct KnowledgeBase {
|
|
| 948 |
}
|
| 949 |
|
| 950 |
for (int depth = 1; depth < def_depth && !frontier.empty(); ++depth){
|
| 951 |
-
|
| 952 |
|
| 953 |
for (StrPtr w : frontier){
|
| 954 |
-
const
|
| 955 |
if (key.empty()) continue;
|
| 956 |
|
| 957 |
auto it2 = global_def_tokens_cache.find(key);
|
|
@@ -966,30 +961,30 @@ struct KnowledgeBase {
|
|
| 966 |
frontier.swap(next_frontier);
|
| 967 |
}
|
| 968 |
|
| 969 |
-
|
| 970 |
out.reserve(acc.size());
|
| 971 |
for (StrPtr p : acc) out.push_back(p);
|
| 972 |
|
| 973 |
{
|
| 974 |
-
|
| 975 |
-
def_index.emplace(wp,
|
| 976 |
}
|
| 977 |
}
|
| 978 |
|
| 979 |
-
|
| 980 |
-
|
| 981 |
auto it = next.find(ctx);
|
| 982 |
-
if (it == next.end()) return
|
| 983 |
return it->second;
|
| 984 |
}
|
| 985 |
};
|
| 986 |
|
| 987 |
-
static void learn_tokens_ngram(KnowledgeBase &kb, const
|
| 988 |
-
|
| 989 |
for (const auto& tok : tokens) {
|
| 990 |
StrPtr tp = kb.interner.intern(tok);
|
| 991 |
for (size_t j = 0; j < window.size(); ++j) {
|
| 992 |
-
|
| 993 |
kb.add_ngram(ctx, tp);
|
| 994 |
}
|
| 995 |
window.push_back(tp);
|
|
@@ -997,10 +992,10 @@ static void learn_tokens_ngram(KnowledgeBase &kb, const std::vector<std::string>
|
|
| 997 |
}
|
| 998 |
}
|
| 999 |
|
| 1000 |
-
static
|
| 1001 |
-
intern_tokens(KnowledgeBase &kb, const
|
| 1002 |
{
|
| 1003 |
-
|
| 1004 |
out.reserve(tokens.size());
|
| 1005 |
for (const auto &t : tokens) out.push_back(kb.interner.intern(t));
|
| 1006 |
return out;
|
|
@@ -1008,9 +1003,9 @@ intern_tokens(KnowledgeBase &kb, const std::vector<std::string> &tokens)
|
|
| 1008 |
|
| 1009 |
static inline bool json_valid_index(size_t i, size_t n){ return i < n; }
|
| 1010 |
|
| 1011 |
-
static
|
| 1012 |
-
|
| 1013 |
-
if (!json_valid_index(i, text.size()) || text[i] != '"') throw
|
| 1014 |
++i;
|
| 1015 |
while (json_valid_index(i, text.size())){
|
| 1016 |
char c = text[i++];
|
|
@@ -1026,14 +1021,14 @@ static std::string parse_quoted_string(const std::string &text, size_t &i){
|
|
| 1026 |
return out;
|
| 1027 |
}
|
| 1028 |
|
| 1029 |
-
static void skip_spaces(const
|
| 1030 |
while (json_valid_index(i, s.size()) && is_space(s[i])) ++i;
|
| 1031 |
}
|
| 1032 |
|
| 1033 |
-
static void skip_json_value(const
|
| 1034 |
|
| 1035 |
-
static
|
| 1036 |
-
|
| 1037 |
if (!json_valid_index(i, text.size()) || text[i] != '[') return out;
|
| 1038 |
|
| 1039 |
++i;
|
|
@@ -1053,7 +1048,7 @@ static std::vector<std::string> parse_json_string_array(const std::string &text,
|
|
| 1053 |
return out;
|
| 1054 |
}
|
| 1055 |
|
| 1056 |
-
static void skip_json_value(const
|
| 1057 |
skip_spaces(s, i);
|
| 1058 |
if (!json_valid_index(i, s.size())) return;
|
| 1059 |
|
|
@@ -1104,11 +1099,11 @@ static void skip_json_value(const std::string &s, size_t &i){
|
|
| 1104 |
}
|
| 1105 |
}
|
| 1106 |
|
| 1107 |
-
static
|
| 1108 |
-
|
| 1109 |
if (dictionary_json_len == 0) return dictionary;
|
| 1110 |
|
| 1111 |
-
|
| 1112 |
text.reserve(dictionary_json_len);
|
| 1113 |
for (unsigned int b = 0; b < dictionary_json_len; ++b){
|
| 1114 |
text.push_back(static_cast<char>(dictionary_json[b]));
|
|
@@ -1139,7 +1134,7 @@ static std::vector<DictionaryEntry> parse_dictionary_json(){
|
|
| 1139 |
if (!json_valid_index(i, text.size())) break;
|
| 1140 |
if (text[i] == '}'){ ++i; break; }
|
| 1141 |
|
| 1142 |
-
|
| 1143 |
skip_spaces(text, i);
|
| 1144 |
if (!json_valid_index(i, text.size()) || text[i] != ':') break;
|
| 1145 |
++i;
|
|
@@ -1160,7 +1155,7 @@ static std::vector<DictionaryEntry> parse_dictionary_json(){
|
|
| 1160 |
if (json_valid_index(i, text.size()) && text[i] == '}'){ ++i; break; }
|
| 1161 |
}
|
| 1162 |
|
| 1163 |
-
if (!entry.word.empty()) dictionary.push_back(
|
| 1164 |
|
| 1165 |
skip_spaces(text, i);
|
| 1166 |
if (json_valid_index(i, text.size()) && text[i] == ','){ ++i; continue; }
|
|
@@ -1170,22 +1165,22 @@ static std::vector<DictionaryEntry> parse_dictionary_json(){
|
|
| 1170 |
return dictionary;
|
| 1171 |
}
|
| 1172 |
|
| 1173 |
-
static
|
| 1174 |
const NextSet &cands,
|
| 1175 |
-
const
|
| 1176 |
-
const
|
| 1177 |
-
const
|
| 1178 |
const StringInterner &interner,
|
| 1179 |
-
const
|
| 1180 |
double repeat_penalty,
|
| 1181 |
-
const
|
| 1182 |
{
|
| 1183 |
-
if (cands.empty()) return
|
| 1184 |
if (cands.size() == 1) return *cands[0];
|
| 1185 |
|
| 1186 |
-
const
|
| 1187 |
|
| 1188 |
-
|
| 1189 |
|
| 1190 |
auto add_token_and_defs = [&](StrPtr t){
|
| 1191 |
if (!t) return;
|
|
@@ -1203,8 +1198,8 @@ static std::string best_candidate_by_similarity(
|
|
| 1203 |
}
|
| 1204 |
|
| 1205 |
// Tokenize the expanded dictionary string to process sub-tokens
|
| 1206 |
-
|
| 1207 |
-
for (const
|
| 1208 |
|
| 1209 |
// Use id_of to check if the sub-token exists without adding a new string to the pool
|
| 1210 |
TokenId sub_tid = interner.id_of(sub_tok);
|
|
@@ -1232,14 +1227,14 @@ static std::string best_candidate_by_similarity(
|
|
| 1232 |
for (StrPtr t : prompt_ptrs) add_token_and_defs(t);
|
| 1233 |
for (StrPtr t : resp_ptrs) add_token_and_defs(t);
|
| 1234 |
|
| 1235 |
-
const
|
| 1236 |
|
| 1237 |
-
const
|
| 1238 |
-
|
| 1239 |
-
|
| 1240 |
|
| 1241 |
-
for (
|
| 1242 |
-
|
| 1243 |
const StrPtr cand = cands[i];
|
| 1244 |
|
| 1245 |
// Tag the candidate word itself
|
|
@@ -1256,8 +1251,8 @@ static std::string best_candidate_by_similarity(
|
|
| 1256 |
}
|
| 1257 |
|
| 1258 |
// Tokenize the candidate to process sub-tokens (e.g., camelCase, hyphens)
|
| 1259 |
-
|
| 1260 |
-
for (const
|
| 1261 |
|
| 1262 |
// Check if the sub-token exists in the knowledge base
|
| 1263 |
TokenId sub_tid = interner.id_of(sub_tok);
|
|
@@ -1285,7 +1280,7 @@ static std::string best_candidate_by_similarity(
|
|
| 1285 |
cand_counts[i] = bitset_count(row, words);
|
| 1286 |
}
|
| 1287 |
|
| 1288 |
-
|
| 1289 |
|
| 1290 |
#if defined(_OPENMP) && defined(CHATIPC_ENABLE_OMP_TARGET)
|
| 1291 |
const bool use_target = (omp_get_num_devices() > 0) && (M >= 256);
|
|
@@ -1294,24 +1289,24 @@ static std::string best_candidate_by_similarity(
|
|
| 1294 |
#endif
|
| 1295 |
|
| 1296 |
if (use_target){
|
| 1297 |
-
|
| 1298 |
-
|
| 1299 |
-
|
| 1300 |
double *score_ptr = scores.data();
|
| 1301 |
-
const
|
| 1302 |
|
| 1303 |
#pragma omp target data map(to: agg_ptr[0:words], cand_ptr[0:cand_words_total], count_ptr[0:M]) map(from: score_ptr[0:M])
|
| 1304 |
{
|
| 1305 |
#pragma omp target teams distribute parallel for
|
| 1306 |
for (ptrdiff_t i = 0; i < static_cast<ptrdiff_t>(M); ++i){
|
| 1307 |
-
const
|
| 1308 |
|
| 1309 |
-
|
| 1310 |
-
for (
|
| 1311 |
inter += popcount_u64(agg_ptr[w] & row[w]);
|
| 1312 |
}
|
| 1313 |
|
| 1314 |
-
const
|
| 1315 |
score_ptr[(size_t)i] = uni ? static_cast<double>(inter) / static_cast<double>(uni) : 0.0;
|
| 1316 |
}
|
| 1317 |
}
|
|
@@ -1320,21 +1315,21 @@ static std::string best_candidate_by_similarity(
|
|
| 1320 |
#pragma omp parallel for schedule(static)
|
| 1321 |
#endif
|
| 1322 |
for (ptrdiff_t i = 0; i < static_cast<ptrdiff_t>(M); ++i){
|
| 1323 |
-
const
|
| 1324 |
|
| 1325 |
-
const
|
| 1326 |
-
const
|
| 1327 |
scores[(size_t)i] = uni ? static_cast<double>(inter) / static_cast<double>(uni) : 0.0;
|
| 1328 |
}
|
| 1329 |
}
|
| 1330 |
|
| 1331 |
double best = -1e9;
|
| 1332 |
-
|
| 1333 |
|
| 1334 |
-
for (
|
| 1335 |
-
const
|
| 1336 |
-
const
|
| 1337 |
-
const
|
| 1338 |
|
| 1339 |
auto rc_it = recent_counts.find(count_key);
|
| 1340 |
const int cnt = (rc_it == recent_counts.end()) ? 0 : rc_it->second;
|
|
@@ -1353,24 +1348,24 @@ static std::string best_candidate_by_similarity(
|
|
| 1353 |
return best_tok;
|
| 1354 |
}
|
| 1355 |
|
| 1356 |
-
static
|
| 1357 |
-
const
|
| 1358 |
size_t response_maxlen,
|
| 1359 |
double repeat_penalty,
|
| 1360 |
int n_gram_size,
|
| 1361 |
bool allow_path_loops)
|
| 1362 |
{
|
| 1363 |
-
|
| 1364 |
if (prompt_toks.empty() || response_maxlen == 0) return resp;
|
| 1365 |
|
| 1366 |
auto prompt_ptrs = intern_tokens(kb, prompt_toks);
|
| 1367 |
-
|
| 1368 |
-
|
| 1369 |
|
| 1370 |
// The context combines prompt and generated text
|
| 1371 |
-
|
| 1372 |
|
| 1373 |
-
auto would_create_2_cycle = [&](const
|
| 1374 |
if (resp.size() < 3) return false;
|
| 1375 |
return normalize_dictionary_key(cand) == normalize_dictionary_key(resp[resp.size() - 2]) &&
|
| 1376 |
normalize_dictionary_key(resp.back()) == normalize_dictionary_key(resp[resp.size() - 3]);
|
|
@@ -1379,11 +1374,11 @@ static std::vector<std::string> construct_response(KnowledgeBase &kb,
|
|
| 1379 |
for (size_t step = 0; step < response_maxlen; ++step){
|
| 1380 |
NextSet candidates;
|
| 1381 |
bool found = false;
|
| 1382 |
-
|
| 1383 |
|
| 1384 |
-
int current_n =
|
| 1385 |
while (current_n > 0) {
|
| 1386 |
-
|
| 1387 |
auto opt = kb.lookup_ngram(search_ctx);
|
| 1388 |
if (opt && !opt->empty()) {
|
| 1389 |
candidates = *opt;
|
|
@@ -1397,8 +1392,8 @@ static std::vector<std::string> construct_response(KnowledgeBase &kb,
|
|
| 1397 |
if (!found || candidates.empty()) break;
|
| 1398 |
|
| 1399 |
if (candidates.size() == 1){
|
| 1400 |
-
|
| 1401 |
-
|
| 1402 |
if (!allow_path_loops && recent_counts[only_key.empty() ? only : only_key] > 0) break;
|
| 1403 |
|
| 1404 |
resp.push_back(only);
|
|
@@ -1406,11 +1401,11 @@ static std::vector<std::string> construct_response(KnowledgeBase &kb,
|
|
| 1406 |
resp_ptrs.push_back(ptr);
|
| 1407 |
context.push_back(ptr);
|
| 1408 |
recent_counts[only_key.empty() ? only : only_key] += 1;
|
| 1409 |
-
|
| 1410 |
continue;
|
| 1411 |
}
|
| 1412 |
|
| 1413 |
-
|
| 1414 |
candidates, prompt_ptrs, resp_ptrs, kb.def_index, kb.interner,
|
| 1415 |
recent_counts, repeat_penalty, context_tok
|
| 1416 |
);
|
|
@@ -1423,249 +1418,249 @@ static std::vector<std::string> construct_response(KnowledgeBase &kb,
|
|
| 1423 |
resp_ptrs.push_back(chosen_ptr);
|
| 1424 |
context.push_back(chosen_ptr);
|
| 1425 |
|
| 1426 |
-
|
| 1427 |
recent_counts[chosen_key.empty() ? chosen : chosen_key] += 1;
|
| 1428 |
|
| 1429 |
-
|
| 1430 |
}
|
| 1431 |
|
| 1432 |
return resp;
|
| 1433 |
}
|
| 1434 |
|
| 1435 |
-
static void learn_from_file(KnowledgeBase &kb, const
|
| 1436 |
-
|
| 1437 |
if (!ifs) return;
|
| 1438 |
-
|
| 1439 |
-
|
| 1440 |
while (ifs >> tok) tokens.push_back(tok);
|
| 1441 |
learn_tokens_ngram(kb, tokens, n_gram_size);
|
| 1442 |
}
|
| 1443 |
|
| 1444 |
-
static void learn_files_parallel(KnowledgeBase &kb, const
|
| 1445 |
#pragma omp parallel for schedule(dynamic)
|
| 1446 |
for (ptrdiff_t i=0;i<static_cast<ptrdiff_t>(files.size());++i){
|
| 1447 |
learn_from_file(kb, files[(size_t)i], n_gram_size);
|
| 1448 |
}
|
| 1449 |
}
|
| 1450 |
|
| 1451 |
-
static constexpr
|
| 1452 |
-
static constexpr
|
| 1453 |
|
| 1454 |
-
static void write_u64(
|
| 1455 |
do {
|
| 1456 |
unsigned char byte = v & 0x7F;
|
| 1457 |
v >>= 7;
|
| 1458 |
if (v != 0) byte |= 0x80;
|
| 1459 |
os.put(static_cast<char>(byte));
|
| 1460 |
} while (v != 0);
|
| 1461 |
-
if (!os) throw
|
| 1462 |
}
|
| 1463 |
|
| 1464 |
-
static
|
| 1465 |
-
|
| 1466 |
unsigned int shift = 0;
|
| 1467 |
unsigned char byte;
|
| 1468 |
do {
|
| 1469 |
if (!is.read(reinterpret_cast<char*>(&byte), 1)) {
|
| 1470 |
-
throw
|
| 1471 |
}
|
| 1472 |
-
v |= (static_cast<
|
| 1473 |
shift += 7;
|
| 1474 |
} while (byte & 0x80);
|
| 1475 |
return v;
|
| 1476 |
}
|
| 1477 |
|
| 1478 |
-
static void write_string(
|
| 1479 |
-
write_u64(os, static_cast<
|
| 1480 |
if (!s.empty()){
|
| 1481 |
-
os.write(s.data(), static_cast<
|
| 1482 |
-
if(!os) throw
|
| 1483 |
}
|
| 1484 |
}
|
| 1485 |
|
| 1486 |
-
static
|
| 1487 |
-
|
| 1488 |
-
//if (n > (1ULL << 30)) throw
|
| 1489 |
|
| 1490 |
-
|
| 1491 |
s.resize(static_cast<size_t>(n));
|
| 1492 |
|
| 1493 |
if (n != 0){
|
| 1494 |
-
is.read(&s[0], static_cast<
|
| 1495 |
-
if(!is) throw
|
| 1496 |
}
|
| 1497 |
return s;
|
| 1498 |
}
|
| 1499 |
|
| 1500 |
-
static void save_kb_binary(const KnowledgeBase &kb, const
|
| 1501 |
-
const
|
| 1502 |
|
| 1503 |
{
|
| 1504 |
-
|
| 1505 |
-
if (!ofs) throw
|
| 1506 |
|
| 1507 |
-
|
| 1508 |
pool.reserve(kb.interner.pool.size());
|
| 1509 |
for (const auto &s : kb.interner.pool) pool.push_back(s);
|
| 1510 |
|
| 1511 |
-
|
| 1512 |
|
| 1513 |
-
|
| 1514 |
id.reserve(pool.size());
|
| 1515 |
-
for (
|
| 1516 |
id.emplace(pool[(size_t)i], i);
|
| 1517 |
}
|
| 1518 |
|
| 1519 |
write_u64(ofs, KB_MAGIC);
|
| 1520 |
write_u64(ofs, KB_VERSION);
|
| 1521 |
-
write_u64(ofs, static_cast<
|
| 1522 |
|
| 1523 |
-
write_u64(ofs, static_cast<
|
| 1524 |
for (const auto &s : pool) write_string(ofs, s);
|
| 1525 |
|
| 1526 |
-
write_u64(ofs, static_cast<
|
| 1527 |
for (const auto &pr : kb.next){
|
| 1528 |
-
write_u64(ofs, static_cast<
|
| 1529 |
for (StrPtr ctx_tok : pr.first){
|
| 1530 |
write_u64(ofs, id.at(*ctx_tok));
|
| 1531 |
}
|
| 1532 |
-
write_u64(ofs, static_cast<
|
| 1533 |
for (StrPtr nxt : pr.second){
|
| 1534 |
write_u64(ofs, id.at(*nxt));
|
| 1535 |
}
|
| 1536 |
}
|
| 1537 |
|
| 1538 |
-
write_u64(ofs, static_cast<
|
| 1539 |
for (const auto &pr : kb.def_index){
|
| 1540 |
write_u64(ofs, id.at(*pr.first));
|
| 1541 |
-
write_u64(ofs, static_cast<
|
| 1542 |
for (StrPtr tok : pr.second){
|
| 1543 |
write_u64(ofs, id.at(*tok));
|
| 1544 |
}
|
| 1545 |
}
|
| 1546 |
|
| 1547 |
ofs.flush();
|
| 1548 |
-
if (!ofs) throw
|
| 1549 |
}
|
| 1550 |
|
| 1551 |
-
|
| 1552 |
-
if (
|
| 1553 |
-
|
| 1554 |
-
throw
|
| 1555 |
}
|
| 1556 |
}
|
| 1557 |
|
| 1558 |
-
static void load_kb_binary(KnowledgeBase &kb, const
|
| 1559 |
-
|
| 1560 |
-
if (!ifs) throw
|
| 1561 |
|
| 1562 |
-
const
|
| 1563 |
-
if (magic != KB_MAGIC) throw
|
| 1564 |
|
| 1565 |
-
const
|
| 1566 |
-
if (version != KB_VERSION) throw
|
| 1567 |
|
| 1568 |
-
const
|
| 1569 |
|
| 1570 |
-
const
|
| 1571 |
-
//if (N > (1ULL << 26)) throw
|
| 1572 |
|
| 1573 |
-
|
| 1574 |
strings.reserve(static_cast<size_t>(N));
|
| 1575 |
|
| 1576 |
-
for (
|
| 1577 |
strings.push_back(read_string(ifs));
|
| 1578 |
}
|
| 1579 |
|
| 1580 |
kb.interner.pool.clear();
|
| 1581 |
kb.interner.pool.reserve(static_cast<size_t>(N));
|
| 1582 |
|
| 1583 |
-
|
| 1584 |
ptrs.reserve(static_cast<size_t>(N));
|
| 1585 |
for (const auto &s : strings){
|
| 1586 |
ptrs.push_back(kb.interner.intern(s));
|
| 1587 |
}
|
| 1588 |
|
| 1589 |
// Rebuild next
|
| 1590 |
-
const
|
| 1591 |
-
//if (E > (1ULL << 26)) throw
|
| 1592 |
|
| 1593 |
{
|
| 1594 |
-
|
| 1595 |
kb.next.clear();
|
| 1596 |
kb.next.reserve(static_cast<size_t>(E));
|
| 1597 |
}
|
| 1598 |
|
| 1599 |
-
for (
|
| 1600 |
-
const
|
| 1601 |
-
//if (ctx_size > (1ULL << 16)) throw
|
| 1602 |
|
| 1603 |
-
|
| 1604 |
ctx.reserve(static_cast<size_t>(ctx_size));
|
| 1605 |
-
for(
|
| 1606 |
-
const
|
| 1607 |
-
if (c_idx >= ptrs.size()) throw
|
| 1608 |
ctx.push_back(ptrs[(size_t)c_idx]);
|
| 1609 |
}
|
| 1610 |
|
| 1611 |
-
const
|
| 1612 |
-
//if (M > (1ULL << 26)) throw
|
| 1613 |
|
| 1614 |
NextSet vec;
|
| 1615 |
vec.reserve(static_cast<size_t>(M));
|
| 1616 |
|
| 1617 |
-
for (
|
| 1618 |
-
const
|
| 1619 |
-
if (v_idx >= ptrs.size()) throw
|
| 1620 |
vec.push_back(ptrs[(size_t)v_idx]);
|
| 1621 |
}
|
| 1622 |
|
| 1623 |
{
|
| 1624 |
-
|
| 1625 |
-
kb.next.emplace(
|
| 1626 |
}
|
| 1627 |
}
|
| 1628 |
|
| 1629 |
// Rebuild def_index from file
|
| 1630 |
-
const
|
| 1631 |
-
//if (K > (1ULL << 26)) throw
|
| 1632 |
|
| 1633 |
{
|
| 1634 |
-
|
| 1635 |
kb.def_index.clear();
|
| 1636 |
kb.def_index.reserve(static_cast<size_t>(K));
|
| 1637 |
kb.def_depth = static_cast<int>(file_def_depth);
|
| 1638 |
}
|
| 1639 |
|
| 1640 |
-
for (
|
| 1641 |
-
const
|
| 1642 |
-
const
|
| 1643 |
|
| 1644 |
-
if (key_idx >= ptrs.size()) throw
|
| 1645 |
-
//if (M > (1ULL << 26)) throw
|
| 1646 |
|
| 1647 |
-
|
| 1648 |
toks.reserve(static_cast<size_t>(M));
|
| 1649 |
|
| 1650 |
-
for (
|
| 1651 |
-
const
|
| 1652 |
-
if (v_idx >= ptrs.size()) throw
|
| 1653 |
toks.push_back(ptrs[(size_t)v_idx]);
|
| 1654 |
}
|
| 1655 |
|
| 1656 |
{
|
| 1657 |
-
|
| 1658 |
-
kb.def_index.emplace(ptrs[(size_t)key_idx],
|
| 1659 |
}
|
| 1660 |
}
|
| 1661 |
|
| 1662 |
if (cli_def_depth != static_cast<int>(file_def_depth)){
|
| 1663 |
kb.set_def_depth(cli_def_depth);
|
| 1664 |
|
| 1665 |
-
|
| 1666 |
targets.reserve(ptrs.size() + kb.next.size() * 2);
|
| 1667 |
|
| 1668 |
-
|
| 1669 |
seen.reserve(ptrs.size() + kb.next.size() * 2);
|
| 1670 |
|
| 1671 |
for (StrPtr p : ptrs){
|
|
@@ -1673,7 +1668,7 @@ static void load_kb_binary(KnowledgeBase &kb, const std::string &fname, int cli_
|
|
| 1673 |
}
|
| 1674 |
|
| 1675 |
{
|
| 1676 |
-
|
| 1677 |
for (const auto &pr : kb.next){
|
| 1678 |
for (StrPtr ctx_element : pr.first) {
|
| 1679 |
if (seen.insert(ctx_element).second) {
|
|
@@ -1696,112 +1691,111 @@ static void load_kb_binary(KnowledgeBase &kb, const std::string &fname, int cli_
|
|
| 1696 |
}
|
| 1697 |
}
|
| 1698 |
|
| 1699 |
-
static void print_kb_info(const
|
| 1700 |
-
|
| 1701 |
if (!ifs) {
|
| 1702 |
-
|
| 1703 |
return;
|
| 1704 |
}
|
| 1705 |
|
| 1706 |
-
|
| 1707 |
|
| 1708 |
try {
|
| 1709 |
-
const
|
| 1710 |
-
if (magic != KB_MAGIC) throw
|
| 1711 |
|
| 1712 |
-
const
|
| 1713 |
-
if (version != KB_VERSION) throw
|
| 1714 |
|
| 1715 |
-
const
|
| 1716 |
|
| 1717 |
-
const
|
| 1718 |
-
//if (N > (1ULL << 26)) throw
|
| 1719 |
|
| 1720 |
// Dynamically read and skip strings since lengths are now variable-length encoded
|
| 1721 |
-
for (
|
| 1722 |
-
|
| 1723 |
-
//if (len > (1ULL << 30)) throw
|
| 1724 |
-
if (len > 0) ifs.seekg(static_cast<
|
| 1725 |
}
|
| 1726 |
|
| 1727 |
-
const
|
| 1728 |
-
//if (E > (1ULL << 26)) throw
|
| 1729 |
|
| 1730 |
-
|
| 1731 |
-
for (
|
| 1732 |
-
const
|
| 1733 |
-
//if (ctx_size > (1ULL << 16)) throw
|
| 1734 |
|
| 1735 |
if (ctx_size > max_ngram) max_ngram = ctx_size;
|
| 1736 |
|
| 1737 |
// Sequentially read (and skip) context keys using LEB128 stream parsing
|
| 1738 |
-
for(
|
| 1739 |
read_u64(ifs);
|
| 1740 |
}
|
| 1741 |
|
| 1742 |
-
const
|
| 1743 |
-
//if (M > (1ULL << 26)) throw
|
| 1744 |
|
| 1745 |
// Sequentially read (and skip) next-node values
|
| 1746 |
-
for(
|
| 1747 |
read_u64(ifs);
|
| 1748 |
}
|
| 1749 |
}
|
| 1750 |
|
| 1751 |
-
|
| 1752 |
|
| 1753 |
-
} catch (const
|
| 1754 |
-
|
| 1755 |
}
|
| 1756 |
}
|
| 1757 |
|
| 1758 |
static void print_commands(const char *p){
|
| 1759 |
-
|
| 1760 |
-
|
| 1761 |
-
|
| 1762 |
-
|
| 1763 |
-
|
| 1764 |
-
|
| 1765 |
-
|
| 1766 |
-
|
| 1767 |
-
|
| 1768 |
-
|
| 1769 |
-
|
| 1770 |
-
|
| 1771 |
-
|
| 1772 |
}
|
| 1773 |
|
| 1774 |
int main(int argc, char **argv){
|
| 1775 |
size_t response_maxlen = 500;
|
| 1776 |
-
|
| 1777 |
-
|
| 1778 |
-
std::string load_kb;
|
| 1779 |
int def_depth = 3;
|
| 1780 |
int n_gram_size = 3;
|
| 1781 |
double repeat_penalty = 0.7; // default λ
|
| 1782 |
-
|
| 1783 |
bool learn_chat = true;
|
| 1784 |
bool allow_path_loops = false;
|
| 1785 |
|
| 1786 |
for (int i=1;i<argc;++i){
|
| 1787 |
-
|
| 1788 |
if (a=="--help"){ print_commands(argv[0]); return 0; }
|
| 1789 |
-
if (a=="--response-max-length" && i+1<argc){ response_maxlen =
|
| 1790 |
if (a=="--save-kb" && i+1<argc){ savefile = argv[++i]; continue; }
|
| 1791 |
if (a=="--load-kb" && i+1<argc){ load_kb = argv[++i]; continue; }
|
| 1792 |
if (a=="--info-kb" && i+1<argc){ print_kb_info(argv[++i]); return 0; }
|
| 1793 |
if (a=="--static-kb"){ learn_chat = false; continue; }
|
| 1794 |
-
if (a=="--dictionary-depth" && i+1<argc){ def_depth =
|
| 1795 |
-
if (a=="--n-gram-max-size" && i+1<argc){ n_gram_size =
|
| 1796 |
if (a=="--learn"){ while(i+1<argc && argv[i+1][0] != '-') learn_files.push_back(argv[++i]); continue; }
|
| 1797 |
-
if (a=="--repeat-penalty" && i+1<argc){ repeat_penalty =
|
| 1798 |
if (a=="--allow-path-loops") { allow_path_loops = true; continue; }
|
| 1799 |
if (a=="--activate-agi"){
|
| 1800 |
-
|
| 1801 |
return 0;
|
| 1802 |
}
|
| 1803 |
|
| 1804 |
-
|
| 1805 |
print_commands(argv[0]);
|
| 1806 |
return 1;
|
| 1807 |
}
|
|
@@ -1813,29 +1807,29 @@ int main(int argc, char **argv){
|
|
| 1813 |
kb.set_def_depth(def_depth);
|
| 1814 |
|
| 1815 |
if (!load_kb.empty()){
|
| 1816 |
-
try {
|
| 1817 |
-
load_kb_binary(kb, load_kb, def_depth);
|
| 1818 |
-
catch (const
|
| 1819 |
}
|
| 1820 |
|
| 1821 |
if (!learn_files.empty()){
|
| 1822 |
-
|
| 1823 |
learn_files_parallel(kb, learn_files, n_gram_size);
|
| 1824 |
}
|
| 1825 |
|
| 1826 |
-
|
| 1827 |
-
|
| 1828 |
-
while (
|
| 1829 |
-
if (line.empty()){
|
| 1830 |
auto prompt_toks = tokenize_whitespace(line);
|
| 1831 |
if (learn_chat) {learn_tokens_ngram(kb, prompt_toks, n_gram_size);}
|
| 1832 |
auto resp = construct_response(kb, prompt_toks, response_maxlen, repeat_penalty, n_gram_size, allow_path_loops);
|
| 1833 |
-
|
| 1834 |
if (!resp.empty() && learn_chat){learn_tokens_ngram(kb, resp, n_gram_size);}
|
| 1835 |
if (!savefile.empty()){
|
| 1836 |
-
try {
|
| 1837 |
-
save_kb_binary(kb, savefile);
|
| 1838 |
-
catch (const
|
| 1839 |
}
|
| 1840 |
}
|
| 1841 |
|
|
|
|
| 1 |
#include <algorithm>
|
|
|
|
| 2 |
#include <cctype>
|
| 3 |
+
#include <cstdint>
|
| 4 |
+
#include <cstdio>
|
| 5 |
#include <fstream>
|
| 6 |
#include <iostream>
|
| 7 |
#include <iterator>
|
|
|
|
| 8 |
#include <mutex>
|
| 9 |
#include <optional>
|
| 10 |
#include <sstream>
|
| 11 |
#include <stdexcept>
|
| 12 |
#include <string>
|
|
|
|
| 13 |
#include <unordered_map>
|
| 14 |
#include <unordered_set>
|
| 15 |
#include <vector>
|
|
|
|
| 18 |
#include <omp.h>
|
| 19 |
#else
|
| 20 |
inline int omp_get_max_threads(){ return 1; }
|
|
|
|
| 21 |
#endif
|
| 22 |
|
| 23 |
extern unsigned char dictionary_json[];
|
| 24 |
extern unsigned int dictionary_json_len;
|
| 25 |
|
| 26 |
+
using namespace std;
|
| 27 |
|
| 28 |
+
static inline bool is_space(char c){ return isspace(static_cast<unsigned char>(c)) != 0; }
|
| 29 |
+
static inline char to_low(char c){ return static_cast<char>(tolower(static_cast<unsigned char>(c))); }
|
|
|
|
| 30 |
|
| 31 |
struct DictionaryEntry {
|
| 32 |
+
string pos;
|
| 33 |
+
string word;
|
| 34 |
+
vector<string> definitions;
|
| 35 |
};
|
| 36 |
|
| 37 |
+
static vector<DictionaryEntry> global_dictionary_entries;
|
| 38 |
+
static unordered_map<string, vector<string>> global_def_tokens_cache;
|
| 39 |
+
static unordered_map<string, vector<string>> global_pos_cache;
|
| 40 |
|
| 41 |
static inline bool is_word_char_for_key(char c){
|
| 42 |
unsigned char uc = static_cast<unsigned char>(c);
|
| 43 |
+
return isalnum(uc) != 0 || c == '\'' || c == '-';
|
| 44 |
}
|
| 45 |
|
| 46 |
+
static string normalize_dictionary_key(const string &s){
|
| 47 |
size_t b = 0, e = s.size();
|
| 48 |
while (b < e && !is_word_char_for_key(s[b])) ++b;
|
| 49 |
while (e > b && !is_word_char_for_key(s[e - 1])) --e;
|
| 50 |
|
| 51 |
+
string out;
|
| 52 |
out.reserve(e - b);
|
| 53 |
for (size_t i = b; i < e; ++i) out.push_back(to_low(s[i]));
|
| 54 |
return out;
|
| 55 |
}
|
| 56 |
|
| 57 |
+
static string normalize_pos_tag(const string &s){
|
| 58 |
+
string out;
|
| 59 |
out.reserve(s.size());
|
| 60 |
for (char c : s){
|
| 61 |
unsigned char uc = static_cast<unsigned char>(c);
|
| 62 |
+
if (isalpha(uc) != 0) out.push_back(to_low(c));
|
| 63 |
}
|
| 64 |
return out;
|
| 65 |
}
|
|
|
|
| 78 |
Interj
|
| 79 |
};
|
| 80 |
|
| 81 |
+
static PosClass pos_class_from_tag(const string &tag){
|
| 82 |
if (tag == "n" || tag == "noun") return PosClass::Noun;
|
| 83 |
if (tag == "v" || tag == "verb" || tag == "part" || tag == "participle" || tag == "p") return PosClass::Verb;
|
| 84 |
if (tag == "a" || tag == "adj" || tag == "adjective") return PosClass::Adj;
|
|
|
|
| 92 |
return PosClass::Unknown;
|
| 93 |
}
|
| 94 |
|
| 95 |
+
static bool has_pos_class(const vector<string> &tags, PosClass cls){
|
| 96 |
for (const auto &t : tags){
|
| 97 |
if (pos_class_from_tag(t) == cls) return true;
|
| 98 |
}
|
| 99 |
return false;
|
| 100 |
}
|
| 101 |
|
| 102 |
+
static const vector<string> &dictionary_pos_for_token(const string &surface){
|
| 103 |
+
static const vector<string> empty;
|
| 104 |
auto key = normalize_dictionary_key(surface);
|
| 105 |
if (key.empty()) return empty;
|
| 106 |
|
|
|
|
| 108 |
return (it == global_pos_cache.end()) ? empty : it->second;
|
| 109 |
}
|
| 110 |
|
| 111 |
+
static bool first_alpha_is_upper(const string &s){
|
| 112 |
for (char c : s){
|
| 113 |
unsigned char uc = static_cast<unsigned char>(c);
|
| 114 |
+
if (isalpha(uc) != 0) return isupper(uc) != 0;
|
| 115 |
}
|
| 116 |
return false;
|
| 117 |
}
|
| 118 |
|
| 119 |
+
static bool first_alpha_is_lower(const string &s){
|
| 120 |
for (char c : s){
|
| 121 |
unsigned char uc = static_cast<unsigned char>(c);
|
| 122 |
+
if (isalpha(uc) != 0) return islower(uc) != 0;
|
| 123 |
}
|
| 124 |
return false;
|
| 125 |
}
|
| 126 |
|
| 127 |
+
static bool is_sentence_boundary_token(const string &s){
|
| 128 |
if (s.empty()) return false;
|
| 129 |
char c = s.back();
|
| 130 |
return c == '.' || c == '!' || c == '?';
|
| 131 |
}
|
| 132 |
|
| 133 |
+
static bool is_open_punct_token(const string &s){
|
| 134 |
return s == "(" || s == "[" || s == "{" || s == "\"" || s == "'";
|
| 135 |
}
|
| 136 |
|
| 137 |
+
static bool is_punctuation_only_token(const string &s){
|
| 138 |
if (s.empty()) return false;
|
| 139 |
for (char c : s){
|
| 140 |
unsigned char uc = static_cast<unsigned char>(c);
|
| 141 |
+
if (isalnum(uc) != 0) return false;
|
| 142 |
}
|
| 143 |
return true;
|
| 144 |
}
|
| 145 |
|
| 146 |
+
static string to_lower_ascii(string s) {
|
| 147 |
+
transform(s.begin(), s.end(), s.begin(),
|
| 148 |
+
[](unsigned char c) { return static_cast<char>(tolower(c)); });
|
| 149 |
return s;
|
| 150 |
}
|
| 151 |
|
| 152 |
+
static bool is_common_determiner(const string &s) {
|
| 153 |
+
static const unordered_set<string> words = {
|
| 154 |
"a", "an", "the", "this", "that", "these", "those",
|
| 155 |
"my", "your", "his", "her", "its", "our", "their",
|
| 156 |
"all", "some", "any", "each", "every", "either", "neither",
|
|
|
|
| 163 |
return words.find(to_lower_ascii(s)) != words.end();
|
| 164 |
}
|
| 165 |
|
| 166 |
+
static bool is_common_preposition(const string &s) {
|
| 167 |
+
static const unordered_set<string> words = {
|
| 168 |
"of", "in", "on", "at", "by", "for", "from", "with",
|
| 169 |
"into", "onto", "about", "over", "under", "after", "before",
|
| 170 |
"between", "through", "during", "without", "within", "across",
|
|
|
|
| 180 |
return words.find(to_lower_ascii(s)) != words.end();
|
| 181 |
}
|
| 182 |
|
| 183 |
+
static bool is_common_aux_or_modal(const string &s) {
|
| 184 |
+
static const unordered_set<string> words = {
|
| 185 |
"to", "be", "am", "is", "are", "was", "were", "been", "being",
|
| 186 |
"have", "has", "had", "do", "does", "did",
|
| 187 |
"can", "could", "may", "might", "must", "shall", "should", "will",
|
|
|
|
| 193 |
return words.find(to_lower_ascii(s)) != words.end();
|
| 194 |
}
|
| 195 |
|
| 196 |
+
static bool begins_with_vowel_sound(const string &s){
|
| 197 |
if (s.empty()) return false;
|
| 198 |
|
| 199 |
+
const string t = to_lower_ascii(s);
|
| 200 |
|
| 201 |
+
static const unordered_set<string> vowel_sound_exceptions = {
|
| 202 |
"hour", "hours", "honest", "honesty", "honor", "honour", "honorable", "honourable",
|
| 203 |
"heir", "heiress", "herb", "herbal", "herbalist", "homage", "hono(u)rary"
|
| 204 |
};
|
| 205 |
|
| 206 |
+
static const unordered_set<string> consonant_sound_prefixes = {
|
| 207 |
"uni", "univ", "unit", "use", "user", "euro", "eu", "one", "once", "ouija",
|
| 208 |
"ufo", "ut", "uk", "ubiq", "ewe", "eul", "eup", "x", "y"
|
| 209 |
};
|
|
|
|
| 220 |
}
|
| 221 |
}
|
| 222 |
|
|
|
|
| 223 |
char c = t[0];
|
| 224 |
return c == 'a' || c == 'e' || c == 'i' || c == 'o' || c == 'u';
|
| 225 |
}
|
| 226 |
|
| 227 |
+
static double english_rule_bonus(const string &context_tok, const string &cand){
|
| 228 |
+
const string ctx_key = normalize_dictionary_key(context_tok);
|
| 229 |
+
const string cand_key = normalize_dictionary_key(cand);
|
| 230 |
|
| 231 |
const auto &ctx_tags = dictionary_pos_for_token(context_tok);
|
| 232 |
const auto &cand_tags = dictionary_pos_for_token(cand);
|
|
|
|
| 287 |
|
| 288 |
if (is_punctuation_only_token(cand)){
|
| 289 |
if (sentence_start) bonus -= 0.05;
|
| 290 |
+
else if (!context_tok.empty() && isalnum(static_cast<unsigned char>(context_tok.back())) != 0) bonus += 0.03;
|
| 291 |
}
|
| 292 |
|
| 293 |
if (is_sentence_boundary_token(cand)) bonus += 0.06;
|
|
|
|
| 295 |
return bonus;
|
| 296 |
}
|
| 297 |
|
| 298 |
+
static vector<string> tokenize_whitespace(const string &s){
|
| 299 |
+
istringstream iss(s);
|
| 300 |
+
vector<string> out;
|
| 301 |
+
string t;
|
| 302 |
while (iss >> t) out.push_back(t);
|
| 303 |
return out;
|
| 304 |
}
|
| 305 |
|
| 306 |
// Static helper to check dictionary presence using the existing pos cache
|
| 307 |
+
static inline bool check_dictionary(const string &word) {
|
| 308 |
return !dictionary_pos_for_token(word).empty();
|
| 309 |
}
|
| 310 |
|
| 311 |
// Morphological lemmatizer to convert sub-tokens to their dictionary-defined base form.
|
| 312 |
+
// Returns optional<string>: contains the valid base form if found in the dictionary,
|
| 313 |
+
// or nullopt if the token has no definition and should be ignored.
|
| 314 |
+
static optional<string> get_valid_base_form(const string &word) {
|
| 315 |
+
if (word.empty()) return nullopt;
|
| 316 |
|
| 317 |
// 1. BEFORE extracting/stemming, check if the original form is already in the dictionary
|
| 318 |
if (check_dictionary(word)) {
|
|
|
|
| 322 |
// 2. Continuous verb form (-ing)
|
| 323 |
if (word.length() > 3 && word.substr(word.length() - 3) == "ing") {
|
| 324 |
// e.g., playing -> play
|
| 325 |
+
string base1 = word.substr(0, word.length() - 3);
|
| 326 |
if (check_dictionary(base1)) return base1;
|
| 327 |
|
| 328 |
// e.g., making -> make
|
| 329 |
+
string base2 = base1 + "e";
|
| 330 |
if (check_dictionary(base2)) return base2;
|
| 331 |
|
| 332 |
// e.g., running -> run (doubled consonant check)
|
| 333 |
if (base1.length() >= 2 && base1.back() == base1[base1.length() - 2]) {
|
| 334 |
+
string base3 = base1.substr(0, base1.length() - 1);
|
| 335 |
if (check_dictionary(base3)) return base3;
|
| 336 |
}
|
| 337 |
}
|
|
|
|
| 339 |
// 3. Past tense / Participle form (-ed)
|
| 340 |
if (word.length() > 2 && word.substr(word.length() - 2) == "ed") {
|
| 341 |
// e.g., baked -> bake
|
| 342 |
+
string base1 = word.substr(0, word.length() - 1);
|
| 343 |
if (check_dictionary(base1)) return base1;
|
| 344 |
|
| 345 |
// e.g., played -> play
|
| 346 |
+
string base2 = word.substr(0, word.length() - 2);
|
| 347 |
if (check_dictionary(base2)) return base2;
|
| 348 |
|
| 349 |
// e.g., hopped -> hop (doubled consonant check)
|
| 350 |
if (base2.length() >= 2 && base2.back() == base2[base2.length() - 2]) {
|
| 351 |
+
string base3 = base2.substr(0, base2.length() - 1);
|
| 352 |
if (check_dictionary(base3)) return base3;
|
| 353 |
}
|
| 354 |
}
|
|
|
|
| 356 |
// 4. Superlative form (-est)
|
| 357 |
if (word.length() > 3 && word.substr(word.length() - 3) == "est") {
|
| 358 |
// e.g., simplest -> simple (strip 'st')
|
| 359 |
+
string base1 = word.substr(0, word.length() - 2);
|
| 360 |
if (check_dictionary(base1)) return base1;
|
| 361 |
|
| 362 |
// e.g., fastest -> fast
|
| 363 |
+
string base2 = word.substr(0, word.length() - 3);
|
| 364 |
if (check_dictionary(base2)) return base2;
|
| 365 |
|
| 366 |
// e.g., happiest -> happy (-iest -> -y)
|
| 367 |
if (word.length() > 4 && word.substr(word.length() - 4) == "iest") {
|
| 368 |
+
string base_y = word.substr(0, word.length() - 4) + "y";
|
| 369 |
if (check_dictionary(base_y)) return base_y;
|
| 370 |
}
|
| 371 |
|
| 372 |
// e.g., biggest -> big (doubled consonant check)
|
| 373 |
if (base2.length() >= 2 && base2.back() == base2[base2.length() - 2]) {
|
| 374 |
+
string base3 = base2.substr(0, base2.length() - 1);
|
| 375 |
if (check_dictionary(base3)) return base3;
|
| 376 |
}
|
| 377 |
}
|
|
|
|
| 379 |
// 5. Comparative form (-er)
|
| 380 |
if (word.length() > 2 && word.substr(word.length() - 2) == "er") {
|
| 381 |
// e.g., simpler -> simple (strip 'r')
|
| 382 |
+
string base1 = word.substr(0, word.length() - 1);
|
| 383 |
if (check_dictionary(base1)) return base1;
|
| 384 |
|
| 385 |
// e.g., faster -> fast
|
| 386 |
+
string base2 = word.substr(0, word.length() - 2);
|
| 387 |
if (check_dictionary(base2)) return base2;
|
| 388 |
|
| 389 |
// e.g., happier -> happy (-ier -> -y)
|
| 390 |
if (word.length() > 3 && word.substr(word.length() - 3) == "ier") {
|
| 391 |
+
string base_y = word.substr(0, word.length() - 3) + "y";
|
| 392 |
if (check_dictionary(base_y)) return base_y;
|
| 393 |
}
|
| 394 |
|
| 395 |
// e.g., bigger -> big (doubled consonant check)
|
| 396 |
if (base2.length() >= 2 && base2.back() == base2[base2.length() - 2]) {
|
| 397 |
+
string base3 = base2.substr(0, base2.length() - 1);
|
| 398 |
if (check_dictionary(base3)) return base3;
|
| 399 |
}
|
| 400 |
}
|
|
|
|
| 402 |
// 6. Adverbial form (-ly)
|
| 403 |
if (word.length() > 2 && word.substr(word.length() - 2) == "ly") {
|
| 404 |
// e.g., quickly -> quick
|
| 405 |
+
string base1 = word.substr(0, word.length() - 2);
|
| 406 |
if (check_dictionary(base1)) return base1;
|
| 407 |
|
| 408 |
// e.g., happily -> happy (-ily -> -y)
|
| 409 |
if (word.length() > 3 && word.substr(word.length() - 3) == "ily") {
|
| 410 |
+
string base2 = word.substr(0, word.length() - 3) + "y";
|
| 411 |
if (check_dictionary(base2)) return base2;
|
| 412 |
}
|
| 413 |
|
| 414 |
// e.g., basically -> basic (-ally -> -ic)
|
| 415 |
if (word.length() > 4 && word.substr(word.length() - 4) == "ally") {
|
| 416 |
+
string base3 = word.substr(0, word.length() - 4);
|
| 417 |
if (check_dictionary(base3)) return base3;
|
| 418 |
|
| 419 |
+
string base4 = base3 + "al";
|
| 420 |
if (check_dictionary(base4)) return base4;
|
| 421 |
}
|
| 422 |
|
| 423 |
// e.g., gently -> gentle (-ly -> -le)
|
| 424 |
if (word.back() == 'y' && word.length() > 2 && word[word.length() - 2] == 'l') {
|
| 425 |
+
string base5 = word.substr(0, word.length() - 1) + "e";
|
| 426 |
if (check_dictionary(base5)) return base5;
|
| 427 |
}
|
| 428 |
}
|
|
|
|
| 430 |
// 7. Noun suffix (-ness)
|
| 431 |
if (word.length() > 4 && word.substr(word.length() - 4) == "ness") {
|
| 432 |
// e.g., sadness -> sad
|
| 433 |
+
string base1 = word.substr(0, word.length() - 4);
|
| 434 |
if (check_dictionary(base1)) return base1;
|
| 435 |
|
| 436 |
// e.g., happiness -> happy (-iness -> -y)
|
| 437 |
if (base1.length() > 1 && base1.back() == 'i') {
|
| 438 |
+
string base2 = base1.substr(0, base1.length() - 1) + "y";
|
| 439 |
if (check_dictionary(base2)) return base2;
|
| 440 |
}
|
| 441 |
}
|
|
|
|
| 443 |
// 8. Noun suffix (-ment)
|
| 444 |
if (word.length() > 4 && word.substr(word.length() - 4) == "ment") {
|
| 445 |
// e.g., development -> develop
|
| 446 |
+
string base1 = word.substr(0, word.length() - 4);
|
| 447 |
if (check_dictionary(base1)) return base1;
|
| 448 |
}
|
| 449 |
|
| 450 |
// 9. Plural / 3rd-person singular form (-ies -> -y)
|
| 451 |
if (word.length() > 3 && word.substr(word.length() - 3) == "ies") {
|
| 452 |
+
string base = word.substr(0, word.length() - 3) + "y";
|
| 453 |
if (check_dictionary(base)) return base;
|
| 454 |
}
|
| 455 |
|
| 456 |
// 10. Plural / 3rd-person singular form (-ves -> -f / -fe)
|
| 457 |
if (word.length() > 3 && word.substr(word.length() - 3) == "ves") {
|
| 458 |
+
string base1 = word.substr(0, word.length() - 3) + "f";
|
| 459 |
if (check_dictionary(base1)) return base1;
|
| 460 |
|
| 461 |
+
string base2 = word.substr(0, word.length() - 3) + "fe";
|
| 462 |
if (check_dictionary(base2)) return base2;
|
| 463 |
}
|
| 464 |
|
| 465 |
// 11. Plural form (-es)
|
| 466 |
if (word.length() > 2 && word.substr(word.length() - 2) == "es") {
|
| 467 |
+
string base = word.substr(0, word.length() - 2);
|
| 468 |
if (check_dictionary(base)) return base;
|
| 469 |
}
|
| 470 |
|
| 471 |
// 12. Plural form (-s)
|
| 472 |
if (word.length() > 1 && word.back() == 's' && word[word.length() - 2] != 's') {
|
| 473 |
+
string base = word.substr(0, word.length() - 1);
|
| 474 |
if (check_dictionary(base)) return base;
|
| 475 |
}
|
| 476 |
|
| 477 |
// No dictionary-defined base form found
|
| 478 |
+
return nullopt;
|
| 479 |
}
|
| 480 |
|
| 481 |
// Helper arrays for number-to-words
|
|
|
|
| 489 |
"quattuordecillion", "quindecillion", "sexdecillion", "septendecillion",
|
| 490 |
"octodecillion", "novemdecillion", "vigintillion", "googol"};
|
| 491 |
|
| 492 |
+
static vector<string> digit_to_words(const string& input_s) {
|
| 493 |
+
vector<string> words;
|
| 494 |
if (input_s.empty()) return words;
|
| 495 |
|
| 496 |
// Split the number into integer and fractional parts
|
| 497 |
+
string int_part = input_s;
|
| 498 |
+
string frac_part = "";
|
| 499 |
size_t dot_pos = input_s.find('.');
|
| 500 |
+
if (dot_pos != string::npos) {
|
| 501 |
int_part = input_s.substr(0, dot_pos);
|
| 502 |
frac_part = input_s.substr(dot_pos + 1);
|
| 503 |
}
|
| 504 |
|
| 505 |
// Clean the integer part by removing commas
|
| 506 |
+
string s;
|
| 507 |
for (char c : int_part) {
|
| 508 |
+
if (isdigit(static_cast<unsigned char>(c))) {
|
| 509 |
s.push_back(c);
|
| 510 |
}
|
| 511 |
}
|
|
|
|
| 515 |
// If there are no integer digits, ensure there are fractional digits before parsing
|
| 516 |
bool has_frac_digits = false;
|
| 517 |
for (char c : frac_part) {
|
| 518 |
+
if (isdigit(static_cast<unsigned char>(c))) has_frac_digits = true;
|
| 519 |
}
|
| 520 |
if (!has_frac_digits) return words;
|
| 521 |
words.push_back("zero");
|
| 522 |
} else {
|
| 523 |
size_t first_non_zero = s.find_first_not_of('0');
|
| 524 |
+
if (first_non_zero == string::npos) {
|
| 525 |
words.push_back("zero");
|
| 526 |
} else {
|
| 527 |
+
string num = s.substr(first_non_zero);
|
| 528 |
|
| 529 |
// Fallback for extremely large numbers to prevent overflow
|
| 530 |
if (num.length() > 69) {
|
|
|
|
| 534 |
} else {
|
| 535 |
int scale_idx = 0;
|
| 536 |
while (!num.empty()) {
|
| 537 |
+
int chunk_len = min<int>(3, num.length());
|
| 538 |
+
string chunk_str = num.substr(num.length() - chunk_len);
|
| 539 |
num = num.substr(0, num.length() - chunk_len);
|
| 540 |
|
| 541 |
+
int chunk = stoi(chunk_str);
|
| 542 |
if (chunk > 0) {
|
| 543 |
+
vector<string> chunk_words;
|
| 544 |
int h = chunk / 100;
|
| 545 |
int remainder = chunk % 100;
|
| 546 |
|
|
|
|
| 576 |
if (!frac_part.empty()) {
|
| 577 |
bool has_frac_digits = false;
|
| 578 |
for (char c : frac_part) {
|
| 579 |
+
if (isdigit(static_cast<unsigned char>(c))) has_frac_digits = true;
|
| 580 |
}
|
| 581 |
if (has_frac_digits) {
|
| 582 |
words.push_back("point");
|
| 583 |
for (char c : frac_part) {
|
| 584 |
+
if (isdigit(static_cast<unsigned char>(c))) {
|
| 585 |
words.push_back(ones[c - '0']);
|
| 586 |
}
|
| 587 |
}
|
|
|
|
| 591 |
return words;
|
| 592 |
}
|
| 593 |
|
| 594 |
+
static vector<string> tokenize_others(const string &s) {
|
| 595 |
if (check_dictionary(s)) {
|
| 596 |
+
string lower_s;
|
| 597 |
lower_s.reserve(s.size());
|
| 598 |
for (char c : s) {
|
| 599 |
lower_s.push_back(to_low(c));
|
|
|
|
| 603 |
|
| 604 |
// Check if the entire string is a formatted number (pure digits, commas, and dots)
|
| 605 |
auto is_num_char = [](unsigned char c) {
|
| 606 |
+
return isdigit(c) != 0 || c == ',' || c == '.';
|
| 607 |
};
|
| 608 |
+
bool is_all_digits = !s.empty() && all_of(s.begin(), s.end(), is_num_char) &&
|
| 609 |
+
any_of(s.begin(), s.end(), [](unsigned char c) { return isdigit(c) != 0; });
|
| 610 |
|
| 611 |
if (is_all_digits) {
|
| 612 |
return digit_to_words(s);
|
| 613 |
}
|
| 614 |
|
| 615 |
+
vector<string> out;
|
| 616 |
+
string cur;
|
| 617 |
|
| 618 |
auto flush = [&]() {
|
| 619 |
if (!cur.empty()) {
|
| 620 |
// Check numbers inside mixed tokens (e.g. "word1,234.56")
|
| 621 |
+
bool cur_all_digits = all_of(cur.begin(), cur.end(), [](unsigned char c) {
|
| 622 |
+
return isdigit(c) != 0 || c == ',' || c == '.';
|
| 623 |
});
|
| 624 |
+
bool has_digits = any_of(cur.begin(), cur.end(), [](unsigned char c) {
|
| 625 |
+
return isdigit(c) != 0;
|
| 626 |
});
|
| 627 |
|
| 628 |
if (cur_all_digits && has_digits) {
|
|
|
|
| 643 |
char ch = static_cast<char>(uc);
|
| 644 |
|
| 645 |
// Check for Operator/Symbol extraction
|
| 646 |
+
if (ispunct(uc) || (isprint(uc) && !isalnum(uc) && !isspace(uc))) {
|
| 647 |
|
| 648 |
// Bypass regular punctuation extraction inside a formatted number
|
| 649 |
if ((ch == ',' || ch == '.') &&
|
| 650 |
!cur.empty() &&
|
| 651 |
+
(isdigit(static_cast<unsigned char>(cur.back())) != 0 || cur.back() == ',' || cur.back() == '.') &&
|
| 652 |
+
(i + 1 < s.size()) && isdigit(static_cast<unsigned char>(s[i+1])) != 0) {
|
| 653 |
cur.push_back(ch);
|
| 654 |
continue;
|
| 655 |
}
|
| 656 |
|
| 657 |
flush();
|
| 658 |
+
string sym(1, ch);
|
| 659 |
|
| 660 |
// Check for double-character operators
|
| 661 |
if (i + 1 < s.size()) {
|
| 662 |
unsigned char nxt_uc = static_cast<unsigned char>(s[i+1]);
|
| 663 |
+
if (ispunct(nxt_uc) || (isprint(nxt_uc) && !isalnum(nxt_uc) && !isspace(nxt_uc))) {
|
| 664 |
+
string sym2 = sym + static_cast<char>(nxt_uc);
|
| 665 |
if (check_dictionary(sym2)) {
|
| 666 |
out.push_back(sym2);
|
| 667 |
++i;
|
|
|
|
| 675 |
continue;
|
| 676 |
}
|
| 677 |
|
| 678 |
+
if (isspace(uc)) {
|
| 679 |
flush();
|
| 680 |
continue;
|
| 681 |
}
|
| 682 |
|
| 683 |
+
const bool is_upper = isupper(uc) != 0;
|
| 684 |
bool camel_boundary = false;
|
| 685 |
if (is_upper && !cur.empty()) {
|
| 686 |
unsigned char prev = static_cast<unsigned char>(s[i - 1]);
|
| 687 |
+
const bool prev_lower_or_digit = (islower(prev) != 0) || (isdigit(prev) != 0);
|
| 688 |
+
const bool prev_upper = isupper(prev) != 0;
|
| 689 |
+
const bool next_lower = (i + 1 < s.size()) && (islower(static_cast<unsigned char>(s[i + 1])) != 0);
|
| 690 |
camel_boundary = prev_lower_or_digit || (prev_upper && next_lower);
|
| 691 |
}
|
| 692 |
|
| 693 |
bool digit_boundary = false;
|
| 694 |
if (!cur.empty()) {
|
| 695 |
unsigned char prev = static_cast<unsigned char>(s[i - 1]);
|
| 696 |
+
bool prev_digit = (isdigit(prev) != 0) || prev == ',' || prev == '.';
|
| 697 |
+
bool curr_digit = isdigit(uc) != 0;
|
| 698 |
if (prev_digit != curr_digit) {
|
| 699 |
digit_boundary = true;
|
| 700 |
}
|
|
|
|
| 704 |
flush();
|
| 705 |
}
|
| 706 |
|
| 707 |
+
if (isalnum(uc) != 0) {
|
| 708 |
+
cur.push_back(static_cast<char>(tolower(uc)));
|
| 709 |
} else {
|
| 710 |
flush();
|
| 711 |
}
|
|
|
|
| 714 |
return out;
|
| 715 |
}
|
| 716 |
|
| 717 |
+
using StrPtr = const string*;
|
| 718 |
+
using TokenId = uint32_t;
|
| 719 |
static constexpr TokenId TOKEN_ID_INVALID = 0xFFFFFFFFu;
|
| 720 |
|
| 721 |
+
static inline size_t popcount_u64(uint64_t x){
|
| 722 |
#if defined(_MSC_VER)
|
| 723 |
+
return static_cast<size_t>(__popcnt64(x));
|
| 724 |
#else
|
| 725 |
+
return static_cast<size_t>(__builtin_popcountll(static_cast<unsigned long long>(x)));
|
| 726 |
#endif
|
| 727 |
}
|
| 728 |
|
| 729 |
struct StringInterner {
|
| 730 |
+
unordered_set<string> pool;
|
| 731 |
+
unordered_map<string, TokenId> id_by_string;
|
| 732 |
+
vector<const string*> string_by_id;
|
| 733 |
+
mutable mutex m;
|
| 734 |
|
| 735 |
+
const string* intern(const string &s){
|
| 736 |
+
lock_guard<mutex> lk(m);
|
| 737 |
|
| 738 |
auto [it, inserted] = pool.emplace(s);
|
| 739 |
if (inserted){
|
|
|
|
| 745 |
return &*it;
|
| 746 |
}
|
| 747 |
|
| 748 |
+
TokenId id_of(const string &s) const {
|
| 749 |
+
lock_guard<mutex> lk(m);
|
| 750 |
|
| 751 |
auto it = id_by_string.find(s);
|
| 752 |
return (it == id_by_string.end()) ? TOKEN_ID_INVALID : it->second;
|
| 753 |
}
|
| 754 |
|
| 755 |
+
TokenId id_of(const string *p) const {
|
| 756 |
return p ? id_of(*p) : TOKEN_ID_INVALID;
|
| 757 |
}
|
| 758 |
|
| 759 |
+
const string* ptr_from_id(TokenId id) const {
|
| 760 |
+
lock_guard<mutex> lk(m);
|
| 761 |
|
| 762 |
return (id < string_by_id.size()) ? string_by_id[(size_t)id] : nullptr;
|
| 763 |
}
|
| 764 |
|
| 765 |
+
size_t size() const {
|
| 766 |
+
lock_guard<mutex> lk(m);
|
| 767 |
return string_by_id.size();
|
| 768 |
}
|
| 769 |
};
|
| 770 |
|
| 771 |
+
static inline void bitset_set(uint64_t *bits, size_t words, TokenId id){
|
| 772 |
if (id == TOKEN_ID_INVALID) return;
|
| 773 |
+
const size_t idx = static_cast<size_t>(id >> 6);
|
| 774 |
if (idx >= words) return;
|
| 775 |
bits[idx] |= (1ULL << (id & 63u));
|
| 776 |
}
|
| 777 |
|
| 778 |
+
static inline size_t bitset_count(const uint64_t *bits, size_t words){
|
| 779 |
+
size_t total = 0;
|
| 780 |
+
for (size_t i = 0; i < words; ++i) total += popcount_u64(bits[i]);
|
| 781 |
return total;
|
| 782 |
}
|
| 783 |
|
| 784 |
+
static inline size_t bitset_intersection_count(const uint64_t *a, const uint64_t *b, size_t words){
|
| 785 |
+
size_t total = 0;
|
| 786 |
+
for (size_t i = 0; i < words; ++i) total += popcount_u64(a[i] & b[i]);
|
| 787 |
return total;
|
| 788 |
}
|
| 789 |
|
|
|
|
| 795 |
|
| 796 |
#pragma omp parallel
|
| 797 |
{
|
| 798 |
+
unordered_map<string, vector<string>> local_def_cache;
|
| 799 |
+
unordered_map<string, vector<string>> local_pos_cache;
|
| 800 |
|
| 801 |
#pragma omp for schedule(guided) nowait
|
| 802 |
for (int i = 0; i < num_entries; ++i){
|
| 803 |
const auto &entry = global_dictionary_entries[i];
|
| 804 |
|
| 805 |
+
vector<string> all_def_toks;
|
| 806 |
for (const auto &def : entry.definitions){
|
| 807 |
auto toks = tokenize_others(def);
|
| 808 |
all_def_toks.insert(all_def_toks.end(), toks.begin(), toks.end());
|
| 809 |
}
|
| 810 |
|
| 811 |
+
string pos = normalize_pos_tag(entry.pos);
|
| 812 |
|
| 813 |
+
vector<string> sub_words;
|
| 814 |
+
string current_sub;
|
| 815 |
for (char c : entry.word) {
|
| 816 |
if (c == ';' || c == ',') {
|
| 817 |
if (!current_sub.empty()) {
|
|
|
|
| 825 |
if (!current_sub.empty()) sub_words.push_back(current_sub);
|
| 826 |
|
| 827 |
for (size_t j = 0; j < sub_words.size(); ++j) {
|
| 828 |
+
const string key = normalize_dictionary_key(sub_words[j]);
|
| 829 |
if (key.empty()) continue;
|
| 830 |
|
| 831 |
if (!pos.empty()) {
|
| 832 |
if (j == sub_words.size() - 1) {
|
| 833 |
+
local_pos_cache[key].push_back(move(pos));
|
| 834 |
} else {
|
| 835 |
local_pos_cache[key].push_back(pos);
|
| 836 |
}
|
|
|
|
| 844 |
{
|
| 845 |
for (auto &pr : local_def_cache) {
|
| 846 |
auto &vec = global_def_tokens_cache[pr.first];
|
| 847 |
+
vec.insert(vec.end(), make_move_iterator(pr.second.begin()), make_move_iterator(pr.second.end()));
|
| 848 |
}
|
| 849 |
for (auto &pr : local_pos_cache) {
|
| 850 |
auto &vec = global_pos_cache[pr.first];
|
| 851 |
+
vec.insert(vec.end(), make_move_iterator(pr.second.begin()), make_move_iterator(pr.second.end()));
|
| 852 |
}
|
| 853 |
}
|
| 854 |
}
|
| 855 |
|
| 856 |
// Extract keys to use OpenMP parallel-for deduplication over hash map structures
|
| 857 |
+
vector<string> def_keys;
|
| 858 |
def_keys.reserve(global_def_tokens_cache.size());
|
| 859 |
for (const auto &pr : global_def_tokens_cache) def_keys.push_back(pr.first);
|
| 860 |
|
| 861 |
#pragma omp parallel for schedule(dynamic)
|
| 862 |
for (ptrdiff_t i = 0; i < static_cast<ptrdiff_t>(def_keys.size()); ++i){
|
| 863 |
auto &v = global_def_tokens_cache[def_keys[i]];
|
| 864 |
+
sort(v.begin(), v.end());
|
| 865 |
+
v.erase(unique(v.begin(), v.end()), v.end());
|
| 866 |
}
|
| 867 |
|
| 868 |
+
vector<string> pos_keys;
|
| 869 |
pos_keys.reserve(global_pos_cache.size());
|
| 870 |
for (const auto &pr : global_pos_cache) pos_keys.push_back(pr.first);
|
| 871 |
|
| 872 |
#pragma omp parallel for schedule(dynamic)
|
| 873 |
for (ptrdiff_t i = 0; i < static_cast<ptrdiff_t>(pos_keys.size()); ++i){
|
| 874 |
auto &v = global_pos_cache[pos_keys[i]];
|
| 875 |
+
sort(v.begin(), v.end());
|
| 876 |
+
v.erase(unique(v.begin(), v.end()), v.end());
|
| 877 |
}
|
| 878 |
}
|
| 879 |
|
| 880 |
+
struct PtrHash { size_t operator()(StrPtr p) const noexcept { return hash<StrPtr>()(p); } };
|
| 881 |
struct PtrEq { bool operator()(StrPtr a, StrPtr b) const noexcept { return a == b; } };
|
| 882 |
|
| 883 |
+
using NextSet = vector<StrPtr>;
|
| 884 |
|
| 885 |
struct NgramHash {
|
| 886 |
+
size_t operator()(const vector<StrPtr>& v) const noexcept {
|
| 887 |
+
size_t seed = v.size();
|
| 888 |
for(auto& p : v) {
|
| 889 |
+
seed ^= hash<StrPtr>()(p) + 0x9e3779b9 + (seed << 6) + (seed >> 2);
|
| 890 |
}
|
| 891 |
return seed;
|
| 892 |
}
|
|
|
|
| 895 |
struct KnowledgeBase {
|
| 896 |
StringInterner interner;
|
| 897 |
// Modified to use a vector of StrPtr (N-gram) as the key
|
| 898 |
+
unordered_map<vector<StrPtr>, NextSet, NgramHash> next;
|
| 899 |
+
mutable mutex m;
|
| 900 |
|
| 901 |
+
unordered_map<StrPtr, vector<StrPtr>, PtrHash, PtrEq> def_index;
|
| 902 |
+
mutable mutex def_m;
|
| 903 |
int def_depth = 0;
|
| 904 |
|
| 905 |
+
void add_ngram(const vector<StrPtr>& ctx, StrPtr v){
|
| 906 |
for (auto p : ctx) ensure_def_for_interned(p);
|
| 907 |
ensure_def_for_interned(v);
|
| 908 |
|
| 909 |
+
lock_guard<mutex> lk(m);
|
| 910 |
auto &vec = next[ctx];
|
| 911 |
for (auto p : vec) if (p == v) return;
|
| 912 |
vec.push_back(v);
|
| 913 |
}
|
| 914 |
|
| 915 |
void set_def_depth(int D){
|
| 916 |
+
lock_guard<mutex> lk(def_m);
|
| 917 |
if (D != def_depth){
|
| 918 |
def_index.clear();
|
| 919 |
def_depth = D;
|
|
|
|
| 924 |
if (wp == nullptr || def_depth <= 0) return;
|
| 925 |
|
| 926 |
{
|
| 927 |
+
lock_guard<mutex> lk(def_m);
|
| 928 |
if (def_index.find(wp) != def_index.end()) return;
|
| 929 |
}
|
| 930 |
|
| 931 |
+
unordered_set<StrPtr, PtrHash, PtrEq> acc;
|
| 932 |
+
vector<StrPtr> frontier;
|
| 933 |
|
| 934 |
+
const string start_key = normalize_dictionary_key(*wp);
|
| 935 |
if (!start_key.empty()){
|
| 936 |
auto it_def = global_def_tokens_cache.find(start_key);
|
| 937 |
if (it_def != global_def_tokens_cache.end()){
|
|
|
|
| 943 |
}
|
| 944 |
|
| 945 |
for (int depth = 1; depth < def_depth && !frontier.empty(); ++depth){
|
| 946 |
+
vector<StrPtr> next_frontier;
|
| 947 |
|
| 948 |
for (StrPtr w : frontier){
|
| 949 |
+
const string key = normalize_dictionary_key(*w);
|
| 950 |
if (key.empty()) continue;
|
| 951 |
|
| 952 |
auto it2 = global_def_tokens_cache.find(key);
|
|
|
|
| 961 |
frontier.swap(next_frontier);
|
| 962 |
}
|
| 963 |
|
| 964 |
+
vector<StrPtr> out;
|
| 965 |
out.reserve(acc.size());
|
| 966 |
for (StrPtr p : acc) out.push_back(p);
|
| 967 |
|
| 968 |
{
|
| 969 |
+
lock_guard<mutex> lk(def_m);
|
| 970 |
+
def_index.emplace(wp, move(out));
|
| 971 |
}
|
| 972 |
}
|
| 973 |
|
| 974 |
+
optional<NextSet> lookup_ngram(const vector<StrPtr>& ctx) const {
|
| 975 |
+
lock_guard<mutex> lk(m);
|
| 976 |
auto it = next.find(ctx);
|
| 977 |
+
if (it == next.end()) return nullopt;
|
| 978 |
return it->second;
|
| 979 |
}
|
| 980 |
};
|
| 981 |
|
| 982 |
+
static void learn_tokens_ngram(KnowledgeBase &kb, const vector<string>& tokens, int n_gram_size) {
|
| 983 |
+
vector<StrPtr> window;
|
| 984 |
for (const auto& tok : tokens) {
|
| 985 |
StrPtr tp = kb.interner.intern(tok);
|
| 986 |
for (size_t j = 0; j < window.size(); ++j) {
|
| 987 |
+
vector<StrPtr> ctx(window.begin() + j, window.end());
|
| 988 |
kb.add_ngram(ctx, tp);
|
| 989 |
}
|
| 990 |
window.push_back(tp);
|
|
|
|
| 992 |
}
|
| 993 |
}
|
| 994 |
|
| 995 |
+
static vector<StrPtr>
|
| 996 |
+
intern_tokens(KnowledgeBase &kb, const vector<string> &tokens)
|
| 997 |
{
|
| 998 |
+
vector<StrPtr> out;
|
| 999 |
out.reserve(tokens.size());
|
| 1000 |
for (const auto &t : tokens) out.push_back(kb.interner.intern(t));
|
| 1001 |
return out;
|
|
|
|
| 1003 |
|
| 1004 |
static inline bool json_valid_index(size_t i, size_t n){ return i < n; }
|
| 1005 |
|
| 1006 |
+
static string parse_quoted_string(const string &text, size_t &i){
|
| 1007 |
+
string out;
|
| 1008 |
+
if (!json_valid_index(i, text.size()) || text[i] != '"') throw runtime_error("expected '\"'");
|
| 1009 |
++i;
|
| 1010 |
while (json_valid_index(i, text.size())){
|
| 1011 |
char c = text[i++];
|
|
|
|
| 1021 |
return out;
|
| 1022 |
}
|
| 1023 |
|
| 1024 |
+
static void skip_spaces(const string &s, size_t &i){
|
| 1025 |
while (json_valid_index(i, s.size()) && is_space(s[i])) ++i;
|
| 1026 |
}
|
| 1027 |
|
| 1028 |
+
static void skip_json_value(const string &s, size_t &i);
|
| 1029 |
|
| 1030 |
+
static vector<string> parse_json_string_array(const string &text, size_t &i){
|
| 1031 |
+
vector<string> out;
|
| 1032 |
if (!json_valid_index(i, text.size()) || text[i] != '[') return out;
|
| 1033 |
|
| 1034 |
++i;
|
|
|
|
| 1048 |
return out;
|
| 1049 |
}
|
| 1050 |
|
| 1051 |
+
static void skip_json_value(const string &s, size_t &i){
|
| 1052 |
skip_spaces(s, i);
|
| 1053 |
if (!json_valid_index(i, s.size())) return;
|
| 1054 |
|
|
|
|
| 1099 |
}
|
| 1100 |
}
|
| 1101 |
|
| 1102 |
+
static vector<DictionaryEntry> parse_dictionary_json(){
|
| 1103 |
+
vector<DictionaryEntry> dictionary;
|
| 1104 |
if (dictionary_json_len == 0) return dictionary;
|
| 1105 |
|
| 1106 |
+
string text;
|
| 1107 |
text.reserve(dictionary_json_len);
|
| 1108 |
for (unsigned int b = 0; b < dictionary_json_len; ++b){
|
| 1109 |
text.push_back(static_cast<char>(dictionary_json[b]));
|
|
|
|
| 1134 |
if (!json_valid_index(i, text.size())) break;
|
| 1135 |
if (text[i] == '}'){ ++i; break; }
|
| 1136 |
|
| 1137 |
+
string field = parse_quoted_string(text, i);
|
| 1138 |
skip_spaces(text, i);
|
| 1139 |
if (!json_valid_index(i, text.size()) || text[i] != ':') break;
|
| 1140 |
++i;
|
|
|
|
| 1155 |
if (json_valid_index(i, text.size()) && text[i] == '}'){ ++i; break; }
|
| 1156 |
}
|
| 1157 |
|
| 1158 |
+
if (!entry.word.empty()) dictionary.push_back(move(entry));
|
| 1159 |
|
| 1160 |
skip_spaces(text, i);
|
| 1161 |
if (json_valid_index(i, text.size()) && text[i] == ','){ ++i; continue; }
|
|
|
|
| 1165 |
return dictionary;
|
| 1166 |
}
|
| 1167 |
|
| 1168 |
+
static string best_candidate_by_similarity(
|
| 1169 |
const NextSet &cands,
|
| 1170 |
+
const vector<StrPtr> &prompt_ptrs,
|
| 1171 |
+
const vector<StrPtr> &resp_ptrs,
|
| 1172 |
+
const unordered_map<StrPtr, vector<StrPtr>, PtrHash, PtrEq> &def_index,
|
| 1173 |
const StringInterner &interner,
|
| 1174 |
+
const unordered_map<string,int> &recent_counts,
|
| 1175 |
double repeat_penalty,
|
| 1176 |
+
const string &context_tok)
|
| 1177 |
{
|
| 1178 |
+
if (cands.empty()) return string();
|
| 1179 |
if (cands.size() == 1) return *cands[0];
|
| 1180 |
|
| 1181 |
+
const size_t words = max<size_t>(1, (interner.size() + 63u) / 64u);
|
| 1182 |
|
| 1183 |
+
vector<uint64_t> agg_words(words, 0ULL);
|
| 1184 |
|
| 1185 |
auto add_token_and_defs = [&](StrPtr t){
|
| 1186 |
if (!t) return;
|
|
|
|
| 1198 |
}
|
| 1199 |
|
| 1200 |
// Tokenize the expanded dictionary string to process sub-tokens
|
| 1201 |
+
vector<string> sub_tokens = tokenize_others(*t);
|
| 1202 |
+
for (const string& sub_tok : sub_tokens) {
|
| 1203 |
|
| 1204 |
// Use id_of to check if the sub-token exists without adding a new string to the pool
|
| 1205 |
TokenId sub_tid = interner.id_of(sub_tok);
|
|
|
|
| 1227 |
for (StrPtr t : prompt_ptrs) add_token_and_defs(t);
|
| 1228 |
for (StrPtr t : resp_ptrs) add_token_and_defs(t);
|
| 1229 |
|
| 1230 |
+
const size_t agg_count = bitset_count(agg_words.data(), words);
|
| 1231 |
|
| 1232 |
+
const size_t M = cands.size();
|
| 1233 |
+
vector<uint64_t> cand_words(M * words, 0ULL);
|
| 1234 |
+
vector<size_t> cand_counts(M, 0);
|
| 1235 |
|
| 1236 |
+
for (size_t i = 0; i < M; ++i){
|
| 1237 |
+
uint64_t *row = cand_words.data() + i * words;
|
| 1238 |
const StrPtr cand = cands[i];
|
| 1239 |
|
| 1240 |
// Tag the candidate word itself
|
|
|
|
| 1251 |
}
|
| 1252 |
|
| 1253 |
// Tokenize the candidate to process sub-tokens (e.g., camelCase, hyphens)
|
| 1254 |
+
vector<string> sub_tokens = tokenize_others(*cand);
|
| 1255 |
+
for (const string& sub_tok : sub_tokens) {
|
| 1256 |
|
| 1257 |
// Check if the sub-token exists in the knowledge base
|
| 1258 |
TokenId sub_tid = interner.id_of(sub_tok);
|
|
|
|
| 1280 |
cand_counts[i] = bitset_count(row, words);
|
| 1281 |
}
|
| 1282 |
|
| 1283 |
+
vector<double> scores(M, 0.0);
|
| 1284 |
|
| 1285 |
#if defined(_OPENMP) && defined(CHATIPC_ENABLE_OMP_TARGET)
|
| 1286 |
const bool use_target = (omp_get_num_devices() > 0) && (M >= 256);
|
|
|
|
| 1289 |
#endif
|
| 1290 |
|
| 1291 |
if (use_target){
|
| 1292 |
+
uint64_t *agg_ptr = agg_words.data();
|
| 1293 |
+
uint64_t *cand_ptr = cand_words.data();
|
| 1294 |
+
size_t *count_ptr = cand_counts.data();
|
| 1295 |
double *score_ptr = scores.data();
|
| 1296 |
+
const size_t cand_words_total = cand_words.size();
|
| 1297 |
|
| 1298 |
#pragma omp target data map(to: agg_ptr[0:words], cand_ptr[0:cand_words_total], count_ptr[0:M]) map(from: score_ptr[0:M])
|
| 1299 |
{
|
| 1300 |
#pragma omp target teams distribute parallel for
|
| 1301 |
for (ptrdiff_t i = 0; i < static_cast<ptrdiff_t>(M); ++i){
|
| 1302 |
+
const uint64_t *row = cand_ptr + static_cast<size_t>(i) * words;
|
| 1303 |
|
| 1304 |
+
size_t inter = 0;
|
| 1305 |
+
for (size_t w = 0; w < words; ++w){
|
| 1306 |
inter += popcount_u64(agg_ptr[w] & row[w]);
|
| 1307 |
}
|
| 1308 |
|
| 1309 |
+
const size_t uni = agg_count + count_ptr[(size_t)i] - inter;
|
| 1310 |
score_ptr[(size_t)i] = uni ? static_cast<double>(inter) / static_cast<double>(uni) : 0.0;
|
| 1311 |
}
|
| 1312 |
}
|
|
|
|
| 1315 |
#pragma omp parallel for schedule(static)
|
| 1316 |
#endif
|
| 1317 |
for (ptrdiff_t i = 0; i < static_cast<ptrdiff_t>(M); ++i){
|
| 1318 |
+
const uint64_t *row = cand_words.data() + static_cast<size_t>(i) * words;
|
| 1319 |
|
| 1320 |
+
const size_t inter = bitset_intersection_count(agg_words.data(), row, words);
|
| 1321 |
+
const size_t uni = agg_count + cand_counts[(size_t)i] - inter;
|
| 1322 |
scores[(size_t)i] = uni ? static_cast<double>(inter) / static_cast<double>(uni) : 0.0;
|
| 1323 |
}
|
| 1324 |
}
|
| 1325 |
|
| 1326 |
double best = -1e9;
|
| 1327 |
+
string best_tok;
|
| 1328 |
|
| 1329 |
+
for (size_t i = 0; i < M; ++i){
|
| 1330 |
+
const string &tok = *cands[i];
|
| 1331 |
+
const string tok_key = normalize_dictionary_key(tok);
|
| 1332 |
+
const string count_key = tok_key.empty() ? tok : tok_key;
|
| 1333 |
|
| 1334 |
auto rc_it = recent_counts.find(count_key);
|
| 1335 |
const int cnt = (rc_it == recent_counts.end()) ? 0 : rc_it->second;
|
|
|
|
| 1348 |
return best_tok;
|
| 1349 |
}
|
| 1350 |
|
| 1351 |
+
static vector<string> construct_response(KnowledgeBase &kb,
|
| 1352 |
+
const vector<string> &prompt_toks,
|
| 1353 |
size_t response_maxlen,
|
| 1354 |
double repeat_penalty,
|
| 1355 |
int n_gram_size,
|
| 1356 |
bool allow_path_loops)
|
| 1357 |
{
|
| 1358 |
+
vector<string> resp;
|
| 1359 |
if (prompt_toks.empty() || response_maxlen == 0) return resp;
|
| 1360 |
|
| 1361 |
auto prompt_ptrs = intern_tokens(kb, prompt_toks);
|
| 1362 |
+
vector<StrPtr> resp_ptrs;
|
| 1363 |
+
unordered_map<string, int> recent_counts;
|
| 1364 |
|
| 1365 |
// The context combines prompt and generated text
|
| 1366 |
+
vector<StrPtr> context = prompt_ptrs;
|
| 1367 |
|
| 1368 |
+
auto would_create_2_cycle = [&](const string &cand) -> bool {
|
| 1369 |
if (resp.size() < 3) return false;
|
| 1370 |
return normalize_dictionary_key(cand) == normalize_dictionary_key(resp[resp.size() - 2]) &&
|
| 1371 |
normalize_dictionary_key(resp.back()) == normalize_dictionary_key(resp[resp.size() - 3]);
|
|
|
|
| 1374 |
for (size_t step = 0; step < response_maxlen; ++step){
|
| 1375 |
NextSet candidates;
|
| 1376 |
bool found = false;
|
| 1377 |
+
string context_tok;
|
| 1378 |
|
| 1379 |
+
int current_n = min(static_cast<int>(context.size()), n_gram_size);
|
| 1380 |
while (current_n > 0) {
|
| 1381 |
+
vector<StrPtr> search_ctx(context.end() - current_n, context.end());
|
| 1382 |
auto opt = kb.lookup_ngram(search_ctx);
|
| 1383 |
if (opt && !opt->empty()) {
|
| 1384 |
candidates = *opt;
|
|
|
|
| 1392 |
if (!found || candidates.empty()) break;
|
| 1393 |
|
| 1394 |
if (candidates.size() == 1){
|
| 1395 |
+
string only = *candidates[0];
|
| 1396 |
+
string only_key = normalize_dictionary_key(only);
|
| 1397 |
if (!allow_path_loops && recent_counts[only_key.empty() ? only : only_key] > 0) break;
|
| 1398 |
|
| 1399 |
resp.push_back(only);
|
|
|
|
| 1401 |
resp_ptrs.push_back(ptr);
|
| 1402 |
context.push_back(ptr);
|
| 1403 |
recent_counts[only_key.empty() ? only : only_key] += 1;
|
| 1404 |
+
cout << only << ' ' << flush;
|
| 1405 |
continue;
|
| 1406 |
}
|
| 1407 |
|
| 1408 |
+
string chosen = best_candidate_by_similarity(
|
| 1409 |
candidates, prompt_ptrs, resp_ptrs, kb.def_index, kb.interner,
|
| 1410 |
recent_counts, repeat_penalty, context_tok
|
| 1411 |
);
|
|
|
|
| 1418 |
resp_ptrs.push_back(chosen_ptr);
|
| 1419 |
context.push_back(chosen_ptr);
|
| 1420 |
|
| 1421 |
+
string chosen_key = normalize_dictionary_key(chosen);
|
| 1422 |
recent_counts[chosen_key.empty() ? chosen : chosen_key] += 1;
|
| 1423 |
|
| 1424 |
+
cout << chosen << ' ' << flush;
|
| 1425 |
}
|
| 1426 |
|
| 1427 |
return resp;
|
| 1428 |
}
|
| 1429 |
|
| 1430 |
+
static void learn_from_file(KnowledgeBase &kb, const string &fname, int n_gram_size){
|
| 1431 |
+
ifstream ifs(fname);
|
| 1432 |
if (!ifs) return;
|
| 1433 |
+
vector<string> tokens;
|
| 1434 |
+
string tok;
|
| 1435 |
while (ifs >> tok) tokens.push_back(tok);
|
| 1436 |
learn_tokens_ngram(kb, tokens, n_gram_size);
|
| 1437 |
}
|
| 1438 |
|
| 1439 |
+
static void learn_files_parallel(KnowledgeBase &kb, const vector<string> &files, int n_gram_size){
|
| 1440 |
#pragma omp parallel for schedule(dynamic)
|
| 1441 |
for (ptrdiff_t i=0;i<static_cast<ptrdiff_t>(files.size());++i){
|
| 1442 |
learn_from_file(kb, files[(size_t)i], n_gram_size);
|
| 1443 |
}
|
| 1444 |
}
|
| 1445 |
|
| 1446 |
+
static constexpr uint64_t KB_MAGIC = 0x434850434B535641ULL;
|
| 1447 |
+
static constexpr uint64_t KB_VERSION = 2ULL;
|
| 1448 |
|
| 1449 |
+
static void write_u64(ostream &os, uint64_t v) {
|
| 1450 |
do {
|
| 1451 |
unsigned char byte = v & 0x7F;
|
| 1452 |
v >>= 7;
|
| 1453 |
if (v != 0) byte |= 0x80;
|
| 1454 |
os.put(static_cast<char>(byte));
|
| 1455 |
} while (v != 0);
|
| 1456 |
+
if (!os) throw runtime_error("write_u64 failed");
|
| 1457 |
}
|
| 1458 |
|
| 1459 |
+
static uint64_t read_u64(istream &is) {
|
| 1460 |
+
uint64_t v = 0;
|
| 1461 |
unsigned int shift = 0;
|
| 1462 |
unsigned char byte;
|
| 1463 |
do {
|
| 1464 |
if (!is.read(reinterpret_cast<char*>(&byte), 1)) {
|
| 1465 |
+
throw runtime_error("read_u64 failed");
|
| 1466 |
}
|
| 1467 |
+
v |= (static_cast<uint64_t>(byte & 0x7F) << shift);
|
| 1468 |
shift += 7;
|
| 1469 |
} while (byte & 0x80);
|
| 1470 |
return v;
|
| 1471 |
}
|
| 1472 |
|
| 1473 |
+
static void write_string(ostream &os, const string &s){
|
| 1474 |
+
write_u64(os, static_cast<uint64_t>(s.size()));
|
| 1475 |
if (!s.empty()){
|
| 1476 |
+
os.write(s.data(), static_cast<streamsize>(s.size()));
|
| 1477 |
+
if(!os) throw runtime_error("write_string failed");
|
| 1478 |
}
|
| 1479 |
}
|
| 1480 |
|
| 1481 |
+
static string read_string(istream &is){
|
| 1482 |
+
uint64_t n = read_u64(is);
|
| 1483 |
+
//if (n > (1ULL << 30)) throw runtime_error("save file is corrupted: string too large");
|
| 1484 |
|
| 1485 |
+
string s;
|
| 1486 |
s.resize(static_cast<size_t>(n));
|
| 1487 |
|
| 1488 |
if (n != 0){
|
| 1489 |
+
is.read(&s[0], static_cast<streamsize>(n));
|
| 1490 |
+
if(!is) throw runtime_error("read_string failed");
|
| 1491 |
}
|
| 1492 |
return s;
|
| 1493 |
}
|
| 1494 |
|
| 1495 |
+
static void save_kb_binary(const KnowledgeBase &kb, const string &fname){
|
| 1496 |
+
const string temp = fname + ".tmp";
|
| 1497 |
|
| 1498 |
{
|
| 1499 |
+
ofstream ofs(temp.c_str(), ios::binary | ios::trunc);
|
| 1500 |
+
if (!ofs) throw runtime_error("cannot open temp save file");
|
| 1501 |
|
| 1502 |
+
vector<string> pool;
|
| 1503 |
pool.reserve(kb.interner.pool.size());
|
| 1504 |
for (const auto &s : kb.interner.pool) pool.push_back(s);
|
| 1505 |
|
| 1506 |
+
sort(pool.begin(), pool.end());
|
| 1507 |
|
| 1508 |
+
unordered_map<string, uint64_t> id;
|
| 1509 |
id.reserve(pool.size());
|
| 1510 |
+
for (uint64_t i = 0; i < static_cast<uint64_t>(pool.size()); ++i){
|
| 1511 |
id.emplace(pool[(size_t)i], i);
|
| 1512 |
}
|
| 1513 |
|
| 1514 |
write_u64(ofs, KB_MAGIC);
|
| 1515 |
write_u64(ofs, KB_VERSION);
|
| 1516 |
+
write_u64(ofs, static_cast<uint64_t>(kb.def_depth));
|
| 1517 |
|
| 1518 |
+
write_u64(ofs, static_cast<uint64_t>(pool.size()));
|
| 1519 |
for (const auto &s : pool) write_string(ofs, s);
|
| 1520 |
|
| 1521 |
+
write_u64(ofs, static_cast<uint64_t>(kb.next.size()));
|
| 1522 |
for (const auto &pr : kb.next){
|
| 1523 |
+
write_u64(ofs, static_cast<uint64_t>(pr.first.size()));
|
| 1524 |
for (StrPtr ctx_tok : pr.first){
|
| 1525 |
write_u64(ofs, id.at(*ctx_tok));
|
| 1526 |
}
|
| 1527 |
+
write_u64(ofs, static_cast<uint64_t>(pr.second.size()));
|
| 1528 |
for (StrPtr nxt : pr.second){
|
| 1529 |
write_u64(ofs, id.at(*nxt));
|
| 1530 |
}
|
| 1531 |
}
|
| 1532 |
|
| 1533 |
+
write_u64(ofs, static_cast<uint64_t>(kb.def_index.size()));
|
| 1534 |
for (const auto &pr : kb.def_index){
|
| 1535 |
write_u64(ofs, id.at(*pr.first));
|
| 1536 |
+
write_u64(ofs, static_cast<uint64_t>(pr.second.size()));
|
| 1537 |
for (StrPtr tok : pr.second){
|
| 1538 |
write_u64(ofs, id.at(*tok));
|
| 1539 |
}
|
| 1540 |
}
|
| 1541 |
|
| 1542 |
ofs.flush();
|
| 1543 |
+
if (!ofs) throw runtime_error("failed while writing temp save file");
|
| 1544 |
}
|
| 1545 |
|
| 1546 |
+
remove(fname.c_str());
|
| 1547 |
+
if (rename(temp.c_str(), fname.c_str()) != 0){
|
| 1548 |
+
remove(temp.c_str());
|
| 1549 |
+
throw runtime_error("failed to commit save file");
|
| 1550 |
}
|
| 1551 |
}
|
| 1552 |
|
| 1553 |
+
static void load_kb_binary(KnowledgeBase &kb, const string &fname, int cli_def_depth){
|
| 1554 |
+
ifstream ifs(fname, ios::binary);
|
| 1555 |
+
if (!ifs) throw runtime_error("cannot open load file");
|
| 1556 |
|
| 1557 |
+
const uint64_t magic = read_u64(ifs);
|
| 1558 |
+
if (magic != KB_MAGIC) throw runtime_error("bad save file magic");
|
| 1559 |
|
| 1560 |
+
const uint64_t version = read_u64(ifs);
|
| 1561 |
+
if (version != KB_VERSION) throw runtime_error("unsupported save file version");
|
| 1562 |
|
| 1563 |
+
const uint64_t file_def_depth = read_u64(ifs);
|
| 1564 |
|
| 1565 |
+
const uint64_t N = read_u64(ifs);
|
| 1566 |
+
//if (N > (1ULL << 26)) throw runtime_error("save file is corrupted: pool too large");
|
| 1567 |
|
| 1568 |
+
vector<string> strings;
|
| 1569 |
strings.reserve(static_cast<size_t>(N));
|
| 1570 |
|
| 1571 |
+
for (uint64_t i = 0; i < N; ++i){
|
| 1572 |
strings.push_back(read_string(ifs));
|
| 1573 |
}
|
| 1574 |
|
| 1575 |
kb.interner.pool.clear();
|
| 1576 |
kb.interner.pool.reserve(static_cast<size_t>(N));
|
| 1577 |
|
| 1578 |
+
vector<StrPtr> ptrs;
|
| 1579 |
ptrs.reserve(static_cast<size_t>(N));
|
| 1580 |
for (const auto &s : strings){
|
| 1581 |
ptrs.push_back(kb.interner.intern(s));
|
| 1582 |
}
|
| 1583 |
|
| 1584 |
// Rebuild next
|
| 1585 |
+
const uint64_t E = read_u64(ifs);
|
| 1586 |
+
//if (E > (1ULL << 26)) throw runtime_error("save file is corrupted: graph too large");
|
| 1587 |
|
| 1588 |
{
|
| 1589 |
+
lock_guard<mutex> lk(kb.m);
|
| 1590 |
kb.next.clear();
|
| 1591 |
kb.next.reserve(static_cast<size_t>(E));
|
| 1592 |
}
|
| 1593 |
|
| 1594 |
+
for (uint64_t i = 0; i < E; ++i){
|
| 1595 |
+
const uint64_t ctx_size = read_u64(ifs);
|
| 1596 |
+
//if (ctx_size > (1ULL << 16)) throw runtime_error("save file is corrupted: n-gram context too large");
|
| 1597 |
|
| 1598 |
+
vector<StrPtr> ctx;
|
| 1599 |
ctx.reserve(static_cast<size_t>(ctx_size));
|
| 1600 |
+
for(uint64_t c = 0; c < ctx_size; ++c) {
|
| 1601 |
+
const uint64_t c_idx = read_u64(ifs);
|
| 1602 |
+
if (c_idx >= ptrs.size()) throw runtime_error("save file is corrupted: bad context key");
|
| 1603 |
ctx.push_back(ptrs[(size_t)c_idx]);
|
| 1604 |
}
|
| 1605 |
|
| 1606 |
+
const uint64_t M = read_u64(ifs);
|
| 1607 |
+
//if (M > (1ULL << 26)) throw runtime_error("save file is corrupted: graph degree too large");
|
| 1608 |
|
| 1609 |
NextSet vec;
|
| 1610 |
vec.reserve(static_cast<size_t>(M));
|
| 1611 |
|
| 1612 |
+
for (uint64_t j = 0; j < M; ++j){
|
| 1613 |
+
const uint64_t v_idx = read_u64(ifs);
|
| 1614 |
+
if (v_idx >= ptrs.size()) throw runtime_error("save file is corrupted: bad graph value");
|
| 1615 |
vec.push_back(ptrs[(size_t)v_idx]);
|
| 1616 |
}
|
| 1617 |
|
| 1618 |
{
|
| 1619 |
+
lock_guard<mutex> lk(kb.m);
|
| 1620 |
+
kb.next.emplace(move(ctx), move(vec));
|
| 1621 |
}
|
| 1622 |
}
|
| 1623 |
|
| 1624 |
// Rebuild def_index from file
|
| 1625 |
+
const uint64_t K = read_u64(ifs);
|
| 1626 |
+
//if (K > (1ULL << 26)) throw runtime_error("save file is corrupted: def_index too large");
|
| 1627 |
|
| 1628 |
{
|
| 1629 |
+
lock_guard<mutex> lk(kb.def_m);
|
| 1630 |
kb.def_index.clear();
|
| 1631 |
kb.def_index.reserve(static_cast<size_t>(K));
|
| 1632 |
kb.def_depth = static_cast<int>(file_def_depth);
|
| 1633 |
}
|
| 1634 |
|
| 1635 |
+
for (uint64_t i = 0; i < K; ++i){
|
| 1636 |
+
const uint64_t key_idx = read_u64(ifs);
|
| 1637 |
+
const uint64_t M = read_u64(ifs);
|
| 1638 |
|
| 1639 |
+
if (key_idx >= ptrs.size()) throw runtime_error("save file is corrupted: bad def key");
|
| 1640 |
+
//if (M > (1ULL << 26)) throw runtime_error("save file is corrupted: def list too large");
|
| 1641 |
|
| 1642 |
+
vector<StrPtr> toks;
|
| 1643 |
toks.reserve(static_cast<size_t>(M));
|
| 1644 |
|
| 1645 |
+
for (uint64_t j = 0; j < M; ++j){
|
| 1646 |
+
const uint64_t v_idx = read_u64(ifs);
|
| 1647 |
+
if (v_idx >= ptrs.size()) throw runtime_error("save file is corrupted: bad def value");
|
| 1648 |
toks.push_back(ptrs[(size_t)v_idx]);
|
| 1649 |
}
|
| 1650 |
|
| 1651 |
{
|
| 1652 |
+
lock_guard<mutex> lk(kb.def_m);
|
| 1653 |
+
kb.def_index.emplace(ptrs[(size_t)key_idx], move(toks));
|
| 1654 |
}
|
| 1655 |
}
|
| 1656 |
|
| 1657 |
if (cli_def_depth != static_cast<int>(file_def_depth)){
|
| 1658 |
kb.set_def_depth(cli_def_depth);
|
| 1659 |
|
| 1660 |
+
vector<StrPtr> targets;
|
| 1661 |
targets.reserve(ptrs.size() + kb.next.size() * 2);
|
| 1662 |
|
| 1663 |
+
unordered_set<StrPtr, PtrHash, PtrEq> seen;
|
| 1664 |
seen.reserve(ptrs.size() + kb.next.size() * 2);
|
| 1665 |
|
| 1666 |
for (StrPtr p : ptrs){
|
|
|
|
| 1668 |
}
|
| 1669 |
|
| 1670 |
{
|
| 1671 |
+
lock_guard<mutex> lk(kb.m);
|
| 1672 |
for (const auto &pr : kb.next){
|
| 1673 |
for (StrPtr ctx_element : pr.first) {
|
| 1674 |
if (seen.insert(ctx_element).second) {
|
|
|
|
| 1691 |
}
|
| 1692 |
}
|
| 1693 |
|
| 1694 |
+
static void print_kb_info(const string &fname) {
|
| 1695 |
+
ifstream ifs(fname, ios::binary);
|
| 1696 |
if (!ifs) {
|
| 1697 |
+
cerr << "Error: cannot open knowledge-base file " << fname << "\n";
|
| 1698 |
return;
|
| 1699 |
}
|
| 1700 |
|
| 1701 |
+
cout << "Reckoning knowledge-base " << fname << " info. ";
|
| 1702 |
|
| 1703 |
try {
|
| 1704 |
+
const uint64_t magic = read_u64(ifs);
|
| 1705 |
+
if (magic != KB_MAGIC) throw runtime_error("bad save file magic");
|
| 1706 |
|
| 1707 |
+
const uint64_t version = read_u64(ifs);
|
| 1708 |
+
if (version != KB_VERSION) throw runtime_error("unsupported save file version");
|
| 1709 |
|
| 1710 |
+
const uint64_t file_def_depth = read_u64(ifs);
|
| 1711 |
|
| 1712 |
+
const uint64_t N = read_u64(ifs);
|
| 1713 |
+
//if (N > (1ULL << 26)) throw runtime_error("save file is corrupted: pool too large");
|
| 1714 |
|
| 1715 |
// Dynamically read and skip strings since lengths are now variable-length encoded
|
| 1716 |
+
for (uint64_t i = 0; i < N; ++i){
|
| 1717 |
+
uint64_t len = read_u64(ifs);
|
| 1718 |
+
//if (len > (1ULL << 30)) throw runtime_error("save file is corrupted: string too large");
|
| 1719 |
+
if (len > 0) ifs.seekg(static_cast<streamoff>(len), ios_base::cur);
|
| 1720 |
}
|
| 1721 |
|
| 1722 |
+
const uint64_t E = read_u64(ifs);
|
| 1723 |
+
//if (E > (1ULL << 26)) throw runtime_error("save file is corrupted: graph too large");
|
| 1724 |
|
| 1725 |
+
uint64_t max_ngram = 0;
|
| 1726 |
+
for (uint64_t i = 0; i < E; ++i){
|
| 1727 |
+
const uint64_t ctx_size = read_u64(ifs);
|
| 1728 |
+
//if (ctx_size > (1ULL << 16)) throw runtime_error("save file is corrupted: n-gram context too large");
|
| 1729 |
|
| 1730 |
if (ctx_size > max_ngram) max_ngram = ctx_size;
|
| 1731 |
|
| 1732 |
// Sequentially read (and skip) context keys using LEB128 stream parsing
|
| 1733 |
+
for(uint64_t c = 0; c < ctx_size; ++c) {
|
| 1734 |
read_u64(ifs);
|
| 1735 |
}
|
| 1736 |
|
| 1737 |
+
const uint64_t M = read_u64(ifs);
|
| 1738 |
+
//if (M > (1ULL << 26)) throw runtime_error("save file is corrupted: graph degree too large");
|
| 1739 |
|
| 1740 |
// Sequentially read (and skip) next-node values
|
| 1741 |
+
for(uint64_t j = 0; j < M; ++j){
|
| 1742 |
read_u64(ifs);
|
| 1743 |
}
|
| 1744 |
}
|
| 1745 |
|
| 1746 |
+
cout << "The " << fname << " dictionary depth is " << file_def_depth << " and " << fname << " n-gram max size is " << max_ngram << "\n";
|
| 1747 |
|
| 1748 |
+
} catch (const exception &e) {
|
| 1749 |
+
cerr << "Error reading knowledge-base info: " << e.what() << "\n";
|
| 1750 |
}
|
| 1751 |
}
|
| 1752 |
|
| 1753 |
static void print_commands(const char *p){
|
| 1754 |
+
cout << p << " command-line interface options:\n";
|
| 1755 |
+
cout << " --response-max-length N Set maximum number of token(s) in a response. Default number is 500.\n";
|
| 1756 |
+
cout << " --save-kb FILE Save the knowledge-base graph to a binary file.\n";
|
| 1757 |
+
cout << " --load-kb FILE Load a previously saved knowledge-base graph from a binary file.\n";
|
| 1758 |
+
cout << " --info-kb FILE Show the dictionary-depth and n-gram max size of a knowledge-base file.\n";
|
| 1759 |
+
cout << " --static-kb Disable learning prompt(s) and response(s) during interactive session.\n";
|
| 1760 |
+
cout << " --dictionary-depth D Set depth of definition expansion used during learning. Default depth is 3.\n";
|
| 1761 |
+
cout << " --n-gram-max-size N Set maximum size of the n-gram where N is the size. Default size is 3.\n";
|
| 1762 |
+
cout << " --learn f1 f2 ... Learn from text file(s) to update the knowledge-base graph.\n";
|
| 1763 |
+
cout << " --repeat-penalty P Set penalty for repeated tokens in response. Default penalty is 0.7 (higher penalty reduce repetition).\n";
|
| 1764 |
+
cout << " --allow-path-loops Allow infinite loops if the knowledge-base graph has a circular single-path reference.\n";
|
| 1765 |
+
cout << " --activate-agi Activate the Artificial General Intelligence (AGI) features.\n";
|
| 1766 |
+
cout << " --help Show " << p << " command-line interface options.\n";
|
| 1767 |
}
|
| 1768 |
|
| 1769 |
int main(int argc, char **argv){
|
| 1770 |
size_t response_maxlen = 500;
|
| 1771 |
+
string savefile;
|
| 1772 |
+
string load_kb;
|
|
|
|
| 1773 |
int def_depth = 3;
|
| 1774 |
int n_gram_size = 3;
|
| 1775 |
double repeat_penalty = 0.7; // default λ
|
| 1776 |
+
vector<string> learn_files;
|
| 1777 |
bool learn_chat = true;
|
| 1778 |
bool allow_path_loops = false;
|
| 1779 |
|
| 1780 |
for (int i=1;i<argc;++i){
|
| 1781 |
+
string a = argv[i];
|
| 1782 |
if (a=="--help"){ print_commands(argv[0]); return 0; }
|
| 1783 |
+
if (a=="--response-max-length" && i+1<argc){ response_maxlen = stoul(argv[++i]); continue; }
|
| 1784 |
if (a=="--save-kb" && i+1<argc){ savefile = argv[++i]; continue; }
|
| 1785 |
if (a=="--load-kb" && i+1<argc){ load_kb = argv[++i]; continue; }
|
| 1786 |
if (a=="--info-kb" && i+1<argc){ print_kb_info(argv[++i]); return 0; }
|
| 1787 |
if (a=="--static-kb"){ learn_chat = false; continue; }
|
| 1788 |
+
if (a=="--dictionary-depth" && i+1<argc){ def_depth = stoi(argv[++i]); continue; }
|
| 1789 |
+
if (a=="--n-gram-max-size" && i+1<argc){ n_gram_size = max(1, stoi(argv[++i])); continue; }
|
| 1790 |
if (a=="--learn"){ while(i+1<argc && argv[i+1][0] != '-') learn_files.push_back(argv[++i]); continue; }
|
| 1791 |
+
if (a=="--repeat-penalty" && i+1<argc){ repeat_penalty = stod(argv[++i]); continue; }
|
| 1792 |
if (a=="--allow-path-loops") { allow_path_loops = true; continue; }
|
| 1793 |
if (a=="--activate-agi"){
|
| 1794 |
+
cerr << "AGI subscription fee is $1,000 CAD per month. To pay the fee, contact Caleb Nwokocha via email cnwokocha@proton.me\n";
|
| 1795 |
return 0;
|
| 1796 |
}
|
| 1797 |
|
| 1798 |
+
cerr << "Error: Unknown or incomplete command-line option '" << a << "'\n";
|
| 1799 |
print_commands(argv[0]);
|
| 1800 |
return 1;
|
| 1801 |
}
|
|
|
|
| 1807 |
kb.set_def_depth(def_depth);
|
| 1808 |
|
| 1809 |
if (!load_kb.empty()){
|
| 1810 |
+
try { cerr << "Loading knowledge base: " << load_kb << "\n";
|
| 1811 |
+
load_kb_binary(kb, load_kb, def_depth); cerr << "Loaded knowledge base: " << load_kb << "\n"; }
|
| 1812 |
+
catch (const exception &e){ cerr << "Error: " << e.what() << "\n"; }
|
| 1813 |
}
|
| 1814 |
|
| 1815 |
if (!learn_files.empty()){
|
| 1816 |
+
cerr << "Learning from file/s (" << learn_files.size() << ") using threads=" << omp_get_max_threads() << "\n";
|
| 1817 |
learn_files_parallel(kb, learn_files, n_gram_size);
|
| 1818 |
}
|
| 1819 |
|
| 1820 |
+
string line;
|
| 1821 |
+
cout << "Ready. Enter prompts.\n";
|
| 1822 |
+
while (cout << "> " , getline(cin, line)){
|
| 1823 |
+
if (line.empty()){ cout << "\n"; continue; }
|
| 1824 |
auto prompt_toks = tokenize_whitespace(line);
|
| 1825 |
if (learn_chat) {learn_tokens_ngram(kb, prompt_toks, n_gram_size);}
|
| 1826 |
auto resp = construct_response(kb, prompt_toks, response_maxlen, repeat_penalty, n_gram_size, allow_path_loops);
|
| 1827 |
+
cout << "\n";
|
| 1828 |
if (!resp.empty() && learn_chat){learn_tokens_ngram(kb, resp, n_gram_size);}
|
| 1829 |
if (!savefile.empty()){
|
| 1830 |
+
try { cerr << "Saving knowledge base: " << savefile << "\n";
|
| 1831 |
+
save_kb_binary(kb, savefile); cerr << "Saved knowledge base: " << savefile << "\n"; }
|
| 1832 |
+
catch (const exception &e){ cerr << "Error: " << e.what() << "\n"; }
|
| 1833 |
}
|
| 1834 |
}
|
| 1835 |
|