English
technician1 commited on
Commit
aa63c12
·
verified ·
1 Parent(s): b91b402

Upload ChatIPC.cpp

Browse files
Files changed (1) hide show
  1. ChatIPC.cpp +415 -421
ChatIPC.cpp CHANGED
@@ -1,18 +1,15 @@
1
  #include <algorithm>
2
- #include <atomic>
3
  #include <cctype>
4
- #include <cinttypes>
5
- #include <cstring>
6
  #include <fstream>
7
  #include <iostream>
8
  #include <iterator>
9
- #include <map>
10
  #include <mutex>
11
  #include <optional>
12
  #include <sstream>
13
  #include <stdexcept>
14
  #include <string>
15
- #include <thread>
16
  #include <unordered_map>
17
  #include <unordered_set>
18
  #include <vector>
@@ -21,49 +18,48 @@
21
  #include <omp.h>
22
  #else
23
  inline int omp_get_max_threads(){ return 1; }
24
- inline int omp_get_thread_num(){ return 0; }
25
  #endif
26
 
27
  extern unsigned char dictionary_json[];
28
  extern unsigned int dictionary_json_len;
29
 
 
30
 
31
- static inline bool is_space(char c){ return std::isspace(static_cast<unsigned char>(c)) != 0; }
32
- static inline char to_low(char c){ return static_cast<char>(std::tolower(static_cast<unsigned char>(c))); }
33
- static inline void safe_flush(std::ostream &os){ os.flush(); }
34
 
35
  struct DictionaryEntry {
36
- std::string pos;
37
- std::string word;
38
- std::vector<std::string> definitions;
39
  };
40
 
41
- static std::vector<DictionaryEntry> global_dictionary_entries;
42
- static std::unordered_map<std::string, std::vector<std::string>> global_def_tokens_cache;
43
- static std::unordered_map<std::string, std::vector<std::string>> global_pos_cache;
44
 
45
  static inline bool is_word_char_for_key(char c){
46
  unsigned char uc = static_cast<unsigned char>(c);
47
- return std::isalnum(uc) != 0 || c == '\'' || c == '-';
48
  }
49
 
50
- static std::string normalize_dictionary_key(const std::string &s){
51
  size_t b = 0, e = s.size();
52
  while (b < e && !is_word_char_for_key(s[b])) ++b;
53
  while (e > b && !is_word_char_for_key(s[e - 1])) --e;
54
 
55
- std::string out;
56
  out.reserve(e - b);
57
  for (size_t i = b; i < e; ++i) out.push_back(to_low(s[i]));
58
  return out;
59
  }
60
 
61
- static std::string normalize_pos_tag(const std::string &s){
62
- std::string out;
63
  out.reserve(s.size());
64
  for (char c : s){
65
  unsigned char uc = static_cast<unsigned char>(c);
66
- if (std::isalpha(uc) != 0) out.push_back(to_low(c));
67
  }
68
  return out;
69
  }
@@ -82,7 +78,7 @@ enum class PosClass {
82
  Interj
83
  };
84
 
85
- static PosClass pos_class_from_tag(const std::string &tag){
86
  if (tag == "n" || tag == "noun") return PosClass::Noun;
87
  if (tag == "v" || tag == "verb" || tag == "part" || tag == "participle" || tag == "p") return PosClass::Verb;
88
  if (tag == "a" || tag == "adj" || tag == "adjective") return PosClass::Adj;
@@ -96,15 +92,15 @@ static PosClass pos_class_from_tag(const std::string &tag){
96
  return PosClass::Unknown;
97
  }
98
 
99
- static bool has_pos_class(const std::vector<std::string> &tags, PosClass cls){
100
  for (const auto &t : tags){
101
  if (pos_class_from_tag(t) == cls) return true;
102
  }
103
  return false;
104
  }
105
 
106
- static const std::vector<std::string> &dictionary_pos_for_token(const std::string &surface){
107
- static const std::vector<std::string> empty;
108
  auto key = normalize_dictionary_key(surface);
109
  if (key.empty()) return empty;
110
 
@@ -112,49 +108,49 @@ static const std::vector<std::string> &dictionary_pos_for_token(const std::strin
112
  return (it == global_pos_cache.end()) ? empty : it->second;
113
  }
114
 
115
- static bool first_alpha_is_upper(const std::string &s){
116
  for (char c : s){
117
  unsigned char uc = static_cast<unsigned char>(c);
118
- if (std::isalpha(uc) != 0) return std::isupper(uc) != 0;
119
  }
120
  return false;
121
  }
122
 
123
- static bool first_alpha_is_lower(const std::string &s){
124
  for (char c : s){
125
  unsigned char uc = static_cast<unsigned char>(c);
126
- if (std::isalpha(uc) != 0) return std::islower(uc) != 0;
127
  }
128
  return false;
129
  }
130
 
131
- static bool is_sentence_boundary_token(const std::string &s){
132
  if (s.empty()) return false;
133
  char c = s.back();
134
  return c == '.' || c == '!' || c == '?';
135
  }
136
 
137
- static bool is_open_punct_token(const std::string &s){
138
  return s == "(" || s == "[" || s == "{" || s == "\"" || s == "'";
139
  }
140
 
141
- static bool is_punctuation_only_token(const std::string &s){
142
  if (s.empty()) return false;
143
  for (char c : s){
144
  unsigned char uc = static_cast<unsigned char>(c);
145
- if (std::isalnum(uc) != 0) return false;
146
  }
147
  return true;
148
  }
149
 
150
- static std::string to_lower_ascii(std::string s) {
151
- std::transform(s.begin(), s.end(), s.begin(),
152
- [](unsigned char c) { return static_cast<char>(std::tolower(c)); });
153
  return s;
154
  }
155
 
156
- static bool is_common_determiner(const std::string &s) {
157
- static const std::unordered_set<std::string> words = {
158
  "a", "an", "the", "this", "that", "these", "those",
159
  "my", "your", "his", "her", "its", "our", "their",
160
  "all", "some", "any", "each", "every", "either", "neither",
@@ -167,8 +163,8 @@ static bool is_common_determiner(const std::string &s) {
167
  return words.find(to_lower_ascii(s)) != words.end();
168
  }
169
 
170
- static bool is_common_preposition(const std::string &s) {
171
- static const std::unordered_set<std::string> words = {
172
  "of", "in", "on", "at", "by", "for", "from", "with",
173
  "into", "onto", "about", "over", "under", "after", "before",
174
  "between", "through", "during", "without", "within", "across",
@@ -184,8 +180,8 @@ static bool is_common_preposition(const std::string &s) {
184
  return words.find(to_lower_ascii(s)) != words.end();
185
  }
186
 
187
- static bool is_common_aux_or_modal(const std::string &s) {
188
- static const std::unordered_set<std::string> words = {
189
  "to", "be", "am", "is", "are", "was", "were", "been", "being",
190
  "have", "has", "had", "do", "does", "did",
191
  "can", "could", "may", "might", "must", "shall", "should", "will",
@@ -197,17 +193,17 @@ static bool is_common_aux_or_modal(const std::string &s) {
197
  return words.find(to_lower_ascii(s)) != words.end();
198
  }
199
 
200
- static bool begins_with_vowel_sound(const std::string &s){
201
  if (s.empty()) return false;
202
 
203
- const std::string t = to_lower_ascii(s);
204
 
205
- static const std::unordered_set<std::string> vowel_sound_exceptions = {
206
  "hour", "hours", "honest", "honesty", "honor", "honour", "honorable", "honourable",
207
  "heir", "heiress", "herb", "herbal", "herbalist", "homage", "hono(u)rary"
208
  };
209
 
210
- static const std::unordered_set<std::string> consonant_sound_prefixes = {
211
  "uni", "univ", "unit", "use", "user", "euro", "eu", "one", "once", "ouija",
212
  "ufo", "ut", "uk", "ubiq", "ewe", "eul", "eup", "x", "y"
213
  };
@@ -224,14 +220,13 @@ static bool begins_with_vowel_sound(const std::string &s){
224
  }
225
  }
226
 
227
- if (t.empty()) return false;
228
  char c = t[0];
229
  return c == 'a' || c == 'e' || c == 'i' || c == 'o' || c == 'u';
230
  }
231
 
232
- static double english_rule_bonus(const std::string &context_tok, const std::string &cand){
233
- const std::string ctx_key = normalize_dictionary_key(context_tok);
234
- const std::string cand_key = normalize_dictionary_key(cand);
235
 
236
  const auto &ctx_tags = dictionary_pos_for_token(context_tok);
237
  const auto &cand_tags = dictionary_pos_for_token(cand);
@@ -292,7 +287,7 @@ static double english_rule_bonus(const std::string &context_tok, const std::stri
292
 
293
  if (is_punctuation_only_token(cand)){
294
  if (sentence_start) bonus -= 0.05;
295
- else if (!context_tok.empty() && std::isalnum(static_cast<unsigned char>(context_tok.back())) != 0) bonus += 0.03;
296
  }
297
 
298
  if (is_sentence_boundary_token(cand)) bonus += 0.06;
@@ -300,24 +295,24 @@ static double english_rule_bonus(const std::string &context_tok, const std::stri
300
  return bonus;
301
  }
302
 
303
- static std::vector<std::string> tokenize_whitespace(const std::string &s){
304
- std::istringstream iss(s);
305
- std::vector<std::string> out;
306
- std::string t;
307
  while (iss >> t) out.push_back(t);
308
  return out;
309
  }
310
 
311
  // Static helper to check dictionary presence using the existing pos cache
312
- static inline bool check_dictionary(const std::string &word) {
313
  return !dictionary_pos_for_token(word).empty();
314
  }
315
 
316
  // Morphological lemmatizer to convert sub-tokens to their dictionary-defined base form.
317
- // Returns std::optional<std::string>: contains the valid base form if found in the dictionary,
318
- // or std::nullopt if the token has no definition and should be ignored.
319
- static std::optional<std::string> get_valid_base_form(const std::string &word) {
320
- if (word.empty()) return std::nullopt;
321
 
322
  // 1. BEFORE extracting/stemming, check if the original form is already in the dictionary
323
  if (check_dictionary(word)) {
@@ -327,16 +322,16 @@ static std::optional<std::string> get_valid_base_form(const std::string &word) {
327
  // 2. Continuous verb form (-ing)
328
  if (word.length() > 3 && word.substr(word.length() - 3) == "ing") {
329
  // e.g., playing -> play
330
- std::string base1 = word.substr(0, word.length() - 3);
331
  if (check_dictionary(base1)) return base1;
332
 
333
  // e.g., making -> make
334
- std::string base2 = base1 + "e";
335
  if (check_dictionary(base2)) return base2;
336
 
337
  // e.g., running -> run (doubled consonant check)
338
  if (base1.length() >= 2 && base1.back() == base1[base1.length() - 2]) {
339
- std::string base3 = base1.substr(0, base1.length() - 1);
340
  if (check_dictionary(base3)) return base3;
341
  }
342
  }
@@ -344,16 +339,16 @@ static std::optional<std::string> get_valid_base_form(const std::string &word) {
344
  // 3. Past tense / Participle form (-ed)
345
  if (word.length() > 2 && word.substr(word.length() - 2) == "ed") {
346
  // e.g., baked -> bake
347
- std::string base1 = word.substr(0, word.length() - 1);
348
  if (check_dictionary(base1)) return base1;
349
 
350
  // e.g., played -> play
351
- std::string base2 = word.substr(0, word.length() - 2);
352
  if (check_dictionary(base2)) return base2;
353
 
354
  // e.g., hopped -> hop (doubled consonant check)
355
  if (base2.length() >= 2 && base2.back() == base2[base2.length() - 2]) {
356
- std::string base3 = base2.substr(0, base2.length() - 1);
357
  if (check_dictionary(base3)) return base3;
358
  }
359
  }
@@ -361,22 +356,22 @@ static std::optional<std::string> get_valid_base_form(const std::string &word) {
361
  // 4. Superlative form (-est)
362
  if (word.length() > 3 && word.substr(word.length() - 3) == "est") {
363
  // e.g., simplest -> simple (strip 'st')
364
- std::string base1 = word.substr(0, word.length() - 2);
365
  if (check_dictionary(base1)) return base1;
366
 
367
  // e.g., fastest -> fast
368
- std::string base2 = word.substr(0, word.length() - 3);
369
  if (check_dictionary(base2)) return base2;
370
 
371
  // e.g., happiest -> happy (-iest -> -y)
372
  if (word.length() > 4 && word.substr(word.length() - 4) == "iest") {
373
- std::string base_y = word.substr(0, word.length() - 4) + "y";
374
  if (check_dictionary(base_y)) return base_y;
375
  }
376
 
377
  // e.g., biggest -> big (doubled consonant check)
378
  if (base2.length() >= 2 && base2.back() == base2[base2.length() - 2]) {
379
- std::string base3 = base2.substr(0, base2.length() - 1);
380
  if (check_dictionary(base3)) return base3;
381
  }
382
  }
@@ -384,22 +379,22 @@ static std::optional<std::string> get_valid_base_form(const std::string &word) {
384
  // 5. Comparative form (-er)
385
  if (word.length() > 2 && word.substr(word.length() - 2) == "er") {
386
  // e.g., simpler -> simple (strip 'r')
387
- std::string base1 = word.substr(0, word.length() - 1);
388
  if (check_dictionary(base1)) return base1;
389
 
390
  // e.g., faster -> fast
391
- std::string base2 = word.substr(0, word.length() - 2);
392
  if (check_dictionary(base2)) return base2;
393
 
394
  // e.g., happier -> happy (-ier -> -y)
395
  if (word.length() > 3 && word.substr(word.length() - 3) == "ier") {
396
- std::string base_y = word.substr(0, word.length() - 3) + "y";
397
  if (check_dictionary(base_y)) return base_y;
398
  }
399
 
400
  // e.g., bigger -> big (doubled consonant check)
401
  if (base2.length() >= 2 && base2.back() == base2[base2.length() - 2]) {
402
- std::string base3 = base2.substr(0, base2.length() - 1);
403
  if (check_dictionary(base3)) return base3;
404
  }
405
  }
@@ -407,27 +402,27 @@ static std::optional<std::string> get_valid_base_form(const std::string &word) {
407
  // 6. Adverbial form (-ly)
408
  if (word.length() > 2 && word.substr(word.length() - 2) == "ly") {
409
  // e.g., quickly -> quick
410
- std::string base1 = word.substr(0, word.length() - 2);
411
  if (check_dictionary(base1)) return base1;
412
 
413
  // e.g., happily -> happy (-ily -> -y)
414
  if (word.length() > 3 && word.substr(word.length() - 3) == "ily") {
415
- std::string base2 = word.substr(0, word.length() - 3) + "y";
416
  if (check_dictionary(base2)) return base2;
417
  }
418
 
419
  // e.g., basically -> basic (-ally -> -ic)
420
  if (word.length() > 4 && word.substr(word.length() - 4) == "ally") {
421
- std::string base3 = word.substr(0, word.length() - 4);
422
  if (check_dictionary(base3)) return base3;
423
 
424
- std::string base4 = base3 + "al";
425
  if (check_dictionary(base4)) return base4;
426
  }
427
 
428
  // e.g., gently -> gentle (-ly -> -le)
429
  if (word.back() == 'y' && word.length() > 2 && word[word.length() - 2] == 'l') {
430
- std::string base5 = word.substr(0, word.length() - 1) + "e";
431
  if (check_dictionary(base5)) return base5;
432
  }
433
  }
@@ -435,12 +430,12 @@ static std::optional<std::string> get_valid_base_form(const std::string &word) {
435
  // 7. Noun suffix (-ness)
436
  if (word.length() > 4 && word.substr(word.length() - 4) == "ness") {
437
  // e.g., sadness -> sad
438
- std::string base1 = word.substr(0, word.length() - 4);
439
  if (check_dictionary(base1)) return base1;
440
 
441
  // e.g., happiness -> happy (-iness -> -y)
442
  if (base1.length() > 1 && base1.back() == 'i') {
443
- std::string base2 = base1.substr(0, base1.length() - 1) + "y";
444
  if (check_dictionary(base2)) return base2;
445
  }
446
  }
@@ -448,39 +443,39 @@ static std::optional<std::string> get_valid_base_form(const std::string &word) {
448
  // 8. Noun suffix (-ment)
449
  if (word.length() > 4 && word.substr(word.length() - 4) == "ment") {
450
  // e.g., development -> develop
451
- std::string base1 = word.substr(0, word.length() - 4);
452
  if (check_dictionary(base1)) return base1;
453
  }
454
 
455
  // 9. Plural / 3rd-person singular form (-ies -> -y)
456
  if (word.length() > 3 && word.substr(word.length() - 3) == "ies") {
457
- std::string base = word.substr(0, word.length() - 3) + "y";
458
  if (check_dictionary(base)) return base;
459
  }
460
 
461
  // 10. Plural / 3rd-person singular form (-ves -> -f / -fe)
462
  if (word.length() > 3 && word.substr(word.length() - 3) == "ves") {
463
- std::string base1 = word.substr(0, word.length() - 3) + "f";
464
  if (check_dictionary(base1)) return base1;
465
 
466
- std::string base2 = word.substr(0, word.length() - 3) + "fe";
467
  if (check_dictionary(base2)) return base2;
468
  }
469
 
470
  // 11. Plural form (-es)
471
  if (word.length() > 2 && word.substr(word.length() - 2) == "es") {
472
- std::string base = word.substr(0, word.length() - 2);
473
  if (check_dictionary(base)) return base;
474
  }
475
 
476
  // 12. Plural form (-s)
477
  if (word.length() > 1 && word.back() == 's' && word[word.length() - 2] != 's') {
478
- std::string base = word.substr(0, word.length() - 1);
479
  if (check_dictionary(base)) return base;
480
  }
481
 
482
  // No dictionary-defined base form found
483
- return std::nullopt;
484
  }
485
 
486
  // Helper arrays for number-to-words
@@ -494,23 +489,23 @@ static const char* const scales[] = {"", "thousand", "million", "billion", "tril
494
  "quattuordecillion", "quindecillion", "sexdecillion", "septendecillion",
495
  "octodecillion", "novemdecillion", "vigintillion", "googol"};
496
 
497
- static std::vector<std::string> digit_to_words(const std::string& input_s) {
498
- std::vector<std::string> words;
499
  if (input_s.empty()) return words;
500
 
501
  // Split the number into integer and fractional parts
502
- std::string int_part = input_s;
503
- std::string frac_part = "";
504
  size_t dot_pos = input_s.find('.');
505
- if (dot_pos != std::string::npos) {
506
  int_part = input_s.substr(0, dot_pos);
507
  frac_part = input_s.substr(dot_pos + 1);
508
  }
509
 
510
  // Clean the integer part by removing commas
511
- std::string s;
512
  for (char c : int_part) {
513
- if (std::isdigit(static_cast<unsigned char>(c))) {
514
  s.push_back(c);
515
  }
516
  }
@@ -520,16 +515,16 @@ static std::vector<std::string> digit_to_words(const std::string& input_s) {
520
  // If there are no integer digits, ensure there are fractional digits before parsing
521
  bool has_frac_digits = false;
522
  for (char c : frac_part) {
523
- if (std::isdigit(static_cast<unsigned char>(c))) has_frac_digits = true;
524
  }
525
  if (!has_frac_digits) return words;
526
  words.push_back("zero");
527
  } else {
528
  size_t first_non_zero = s.find_first_not_of('0');
529
- if (first_non_zero == std::string::npos) {
530
  words.push_back("zero");
531
  } else {
532
- std::string num = s.substr(first_non_zero);
533
 
534
  // Fallback for extremely large numbers to prevent overflow
535
  if (num.length() > 69) {
@@ -539,13 +534,13 @@ static std::vector<std::string> digit_to_words(const std::string& input_s) {
539
  } else {
540
  int scale_idx = 0;
541
  while (!num.empty()) {
542
- int chunk_len = std::min<int>(3, num.length());
543
- std::string chunk_str = num.substr(num.length() - chunk_len);
544
  num = num.substr(0, num.length() - chunk_len);
545
 
546
- int chunk = std::stoi(chunk_str);
547
  if (chunk > 0) {
548
- std::vector<std::string> chunk_words;
549
  int h = chunk / 100;
550
  int remainder = chunk % 100;
551
 
@@ -581,12 +576,12 @@ static std::vector<std::string> digit_to_words(const std::string& input_s) {
581
  if (!frac_part.empty()) {
582
  bool has_frac_digits = false;
583
  for (char c : frac_part) {
584
- if (std::isdigit(static_cast<unsigned char>(c))) has_frac_digits = true;
585
  }
586
  if (has_frac_digits) {
587
  words.push_back("point");
588
  for (char c : frac_part) {
589
- if (std::isdigit(static_cast<unsigned char>(c))) {
590
  words.push_back(ones[c - '0']);
591
  }
592
  }
@@ -596,9 +591,9 @@ static std::vector<std::string> digit_to_words(const std::string& input_s) {
596
  return words;
597
  }
598
 
599
- static std::vector<std::string> tokenize_others(const std::string &s) {
600
  if (check_dictionary(s)) {
601
- std::string lower_s;
602
  lower_s.reserve(s.size());
603
  for (char c : s) {
604
  lower_s.push_back(to_low(c));
@@ -608,26 +603,26 @@ static std::vector<std::string> tokenize_others(const std::string &s) {
608
 
609
  // Check if the entire string is a formatted number (pure digits, commas, and dots)
610
  auto is_num_char = [](unsigned char c) {
611
- return std::isdigit(c) != 0 || c == ',' || c == '.';
612
  };
613
- bool is_all_digits = !s.empty() && std::all_of(s.begin(), s.end(), is_num_char) &&
614
- std::any_of(s.begin(), s.end(), [](unsigned char c) { return std::isdigit(c) != 0; });
615
 
616
  if (is_all_digits) {
617
  return digit_to_words(s);
618
  }
619
 
620
- std::vector<std::string> out;
621
- std::string cur;
622
 
623
  auto flush = [&]() {
624
  if (!cur.empty()) {
625
  // Check numbers inside mixed tokens (e.g. "word1,234.56")
626
- bool cur_all_digits = std::all_of(cur.begin(), cur.end(), [](unsigned char c) {
627
- return std::isdigit(c) != 0 || c == ',' || c == '.';
628
  });
629
- bool has_digits = std::any_of(cur.begin(), cur.end(), [](unsigned char c) {
630
- return std::isdigit(c) != 0;
631
  });
632
 
633
  if (cur_all_digits && has_digits) {
@@ -648,25 +643,25 @@ static std::vector<std::string> tokenize_others(const std::string &s) {
648
  char ch = static_cast<char>(uc);
649
 
650
  // Check for Operator/Symbol extraction
651
- if (std::ispunct(uc) || (std::isprint(uc) && !std::isalnum(uc) && !std::isspace(uc))) {
652
 
653
  // Bypass regular punctuation extraction inside a formatted number
654
  if ((ch == ',' || ch == '.') &&
655
  !cur.empty() &&
656
- (std::isdigit(static_cast<unsigned char>(cur.back())) != 0 || cur.back() == ',' || cur.back() == '.') &&
657
- (i + 1 < s.size()) && std::isdigit(static_cast<unsigned char>(s[i+1])) != 0) {
658
  cur.push_back(ch);
659
  continue;
660
  }
661
 
662
  flush();
663
- std::string sym(1, ch);
664
 
665
  // Check for double-character operators
666
  if (i + 1 < s.size()) {
667
  unsigned char nxt_uc = static_cast<unsigned char>(s[i+1]);
668
- if (std::ispunct(nxt_uc) || (std::isprint(nxt_uc) && !std::isalnum(nxt_uc) && !std::isspace(nxt_uc))) {
669
- std::string sym2 = sym + static_cast<char>(nxt_uc);
670
  if (check_dictionary(sym2)) {
671
  out.push_back(sym2);
672
  ++i;
@@ -680,26 +675,26 @@ static std::vector<std::string> tokenize_others(const std::string &s) {
680
  continue;
681
  }
682
 
683
- if (std::isspace(uc)) {
684
  flush();
685
  continue;
686
  }
687
 
688
- const bool is_upper = std::isupper(uc) != 0;
689
  bool camel_boundary = false;
690
  if (is_upper && !cur.empty()) {
691
  unsigned char prev = static_cast<unsigned char>(s[i - 1]);
692
- const bool prev_lower_or_digit = (std::islower(prev) != 0) || (std::isdigit(prev) != 0);
693
- const bool prev_upper = std::isupper(prev) != 0;
694
- const bool next_lower = (i + 1 < s.size()) && (std::islower(static_cast<unsigned char>(s[i + 1])) != 0);
695
  camel_boundary = prev_lower_or_digit || (prev_upper && next_lower);
696
  }
697
 
698
  bool digit_boundary = false;
699
  if (!cur.empty()) {
700
  unsigned char prev = static_cast<unsigned char>(s[i - 1]);
701
- bool prev_digit = (std::isdigit(prev) != 0) || prev == ',' || prev == '.';
702
- bool curr_digit = std::isdigit(uc) != 0;
703
  if (prev_digit != curr_digit) {
704
  digit_boundary = true;
705
  }
@@ -709,8 +704,8 @@ static std::vector<std::string> tokenize_others(const std::string &s) {
709
  flush();
710
  }
711
 
712
- if (std::isalnum(uc) != 0) {
713
- cur.push_back(static_cast<char>(std::tolower(uc)));
714
  } else {
715
  flush();
716
  }
@@ -719,26 +714,26 @@ static std::vector<std::string> tokenize_others(const std::string &s) {
719
  return out;
720
  }
721
 
722
- using StrPtr = const std::string*;
723
- using TokenId = std::uint32_t;
724
  static constexpr TokenId TOKEN_ID_INVALID = 0xFFFFFFFFu;
725
 
726
- static inline std::size_t popcount_u64(std::uint64_t x){
727
  #if defined(_MSC_VER)
728
- return static_cast<std::size_t>(__popcnt64(x));
729
  #else
730
- return static_cast<std::size_t>(__builtin_popcountll(static_cast<unsigned long long>(x)));
731
  #endif
732
  }
733
 
734
  struct StringInterner {
735
- std::unordered_set<std::string> pool;
736
- std::unordered_map<std::string, TokenId> id_by_string;
737
- std::vector<const std::string*> string_by_id;
738
- mutable std::mutex m;
739
 
740
- const std::string* intern(const std::string &s){
741
- std::lock_guard<std::mutex> lk(m);
742
 
743
  auto [it, inserted] = pool.emplace(s);
744
  if (inserted){
@@ -750,45 +745,45 @@ struct StringInterner {
750
  return &*it;
751
  }
752
 
753
- TokenId id_of(const std::string &s) const {
754
- std::lock_guard<std::mutex> lk(m);
755
 
756
  auto it = id_by_string.find(s);
757
  return (it == id_by_string.end()) ? TOKEN_ID_INVALID : it->second;
758
  }
759
 
760
- TokenId id_of(const std::string *p) const {
761
  return p ? id_of(*p) : TOKEN_ID_INVALID;
762
  }
763
 
764
- const std::string* ptr_from_id(TokenId id) const {
765
- std::lock_guard<std::mutex> lk(m);
766
 
767
  return (id < string_by_id.size()) ? string_by_id[(size_t)id] : nullptr;
768
  }
769
 
770
- std::size_t size() const {
771
- std::lock_guard<std::mutex> lk(m);
772
  return string_by_id.size();
773
  }
774
  };
775
 
776
- static inline void bitset_set(std::uint64_t *bits, std::size_t words, TokenId id){
777
  if (id == TOKEN_ID_INVALID) return;
778
- const std::size_t idx = static_cast<std::size_t>(id >> 6);
779
  if (idx >= words) return;
780
  bits[idx] |= (1ULL << (id & 63u));
781
  }
782
 
783
- static inline std::size_t bitset_count(const std::uint64_t *bits, std::size_t words){
784
- std::size_t total = 0;
785
- for (std::size_t i = 0; i < words; ++i) total += popcount_u64(bits[i]);
786
  return total;
787
  }
788
 
789
- static inline std::size_t bitset_intersection_count(const std::uint64_t *a, const std::uint64_t *b, std::size_t words){
790
- std::size_t total = 0;
791
- for (std::size_t i = 0; i < words; ++i) total += popcount_u64(a[i] & b[i]);
792
  return total;
793
  }
794
 
@@ -800,23 +795,23 @@ static void build_def_tokens_cache(){
800
 
801
  #pragma omp parallel
802
  {
803
- std::unordered_map<std::string, std::vector<std::string>> local_def_cache;
804
- std::unordered_map<std::string, std::vector<std::string>> local_pos_cache;
805
 
806
  #pragma omp for schedule(guided) nowait
807
  for (int i = 0; i < num_entries; ++i){
808
  const auto &entry = global_dictionary_entries[i];
809
 
810
- std::vector<std::string> all_def_toks;
811
  for (const auto &def : entry.definitions){
812
  auto toks = tokenize_others(def);
813
  all_def_toks.insert(all_def_toks.end(), toks.begin(), toks.end());
814
  }
815
 
816
- std::string pos = normalize_pos_tag(entry.pos);
817
 
818
- std::vector<std::string> sub_words;
819
- std::string current_sub;
820
  for (char c : entry.word) {
821
  if (c == ';' || c == ',') {
822
  if (!current_sub.empty()) {
@@ -830,12 +825,12 @@ static void build_def_tokens_cache(){
830
  if (!current_sub.empty()) sub_words.push_back(current_sub);
831
 
832
  for (size_t j = 0; j < sub_words.size(); ++j) {
833
- const std::string key = normalize_dictionary_key(sub_words[j]);
834
  if (key.empty()) continue;
835
 
836
  if (!pos.empty()) {
837
  if (j == sub_words.size() - 1) {
838
- local_pos_cache[key].push_back(std::move(pos));
839
  } else {
840
  local_pos_cache[key].push_back(pos);
841
  }
@@ -849,49 +844,49 @@ static void build_def_tokens_cache(){
849
  {
850
  for (auto &pr : local_def_cache) {
851
  auto &vec = global_def_tokens_cache[pr.first];
852
- vec.insert(vec.end(), std::make_move_iterator(pr.second.begin()), std::make_move_iterator(pr.second.end()));
853
  }
854
  for (auto &pr : local_pos_cache) {
855
  auto &vec = global_pos_cache[pr.first];
856
- vec.insert(vec.end(), std::make_move_iterator(pr.second.begin()), std::make_move_iterator(pr.second.end()));
857
  }
858
  }
859
  }
860
 
861
  // Extract keys to use OpenMP parallel-for deduplication over hash map structures
862
- std::vector<std::string> def_keys;
863
  def_keys.reserve(global_def_tokens_cache.size());
864
  for (const auto &pr : global_def_tokens_cache) def_keys.push_back(pr.first);
865
 
866
  #pragma omp parallel for schedule(dynamic)
867
  for (ptrdiff_t i = 0; i < static_cast<ptrdiff_t>(def_keys.size()); ++i){
868
  auto &v = global_def_tokens_cache[def_keys[i]];
869
- std::sort(v.begin(), v.end());
870
- v.erase(std::unique(v.begin(), v.end()), v.end());
871
  }
872
 
873
- std::vector<std::string> pos_keys;
874
  pos_keys.reserve(global_pos_cache.size());
875
  for (const auto &pr : global_pos_cache) pos_keys.push_back(pr.first);
876
 
877
  #pragma omp parallel for schedule(dynamic)
878
  for (ptrdiff_t i = 0; i < static_cast<ptrdiff_t>(pos_keys.size()); ++i){
879
  auto &v = global_pos_cache[pos_keys[i]];
880
- std::sort(v.begin(), v.end());
881
- v.erase(std::unique(v.begin(), v.end()), v.end());
882
  }
883
  }
884
 
885
- struct PtrHash { size_t operator()(StrPtr p) const noexcept { return std::hash<StrPtr>()(p); } };
886
  struct PtrEq { bool operator()(StrPtr a, StrPtr b) const noexcept { return a == b; } };
887
 
888
- using NextSet = std::vector<StrPtr>;
889
 
890
  struct NgramHash {
891
- std::size_t operator()(const std::vector<StrPtr>& v) const noexcept {
892
- std::size_t seed = v.size();
893
  for(auto& p : v) {
894
- seed ^= std::hash<StrPtr>()(p) + 0x9e3779b9 + (seed << 6) + (seed >> 2);
895
  }
896
  return seed;
897
  }
@@ -900,25 +895,25 @@ struct NgramHash {
900
  struct KnowledgeBase {
901
  StringInterner interner;
902
  // Modified to use a vector of StrPtr (N-gram) as the key
903
- std::unordered_map<std::vector<StrPtr>, NextSet, NgramHash> next;
904
- mutable std::mutex m;
905
 
906
- std::unordered_map<StrPtr, std::vector<StrPtr>, PtrHash, PtrEq> def_index;
907
- mutable std::mutex def_m;
908
  int def_depth = 0;
909
 
910
- void add_ngram(const std::vector<StrPtr>& ctx, StrPtr v){
911
  for (auto p : ctx) ensure_def_for_interned(p);
912
  ensure_def_for_interned(v);
913
 
914
- std::lock_guard<std::mutex> lk(m);
915
  auto &vec = next[ctx];
916
  for (auto p : vec) if (p == v) return;
917
  vec.push_back(v);
918
  }
919
 
920
  void set_def_depth(int D){
921
- std::lock_guard<std::mutex> lk(def_m);
922
  if (D != def_depth){
923
  def_index.clear();
924
  def_depth = D;
@@ -929,14 +924,14 @@ struct KnowledgeBase {
929
  if (wp == nullptr || def_depth <= 0) return;
930
 
931
  {
932
- std::lock_guard<std::mutex> lk(def_m);
933
  if (def_index.find(wp) != def_index.end()) return;
934
  }
935
 
936
- std::unordered_set<StrPtr, PtrHash, PtrEq> acc;
937
- std::vector<StrPtr> frontier;
938
 
939
- const std::string start_key = normalize_dictionary_key(*wp);
940
  if (!start_key.empty()){
941
  auto it_def = global_def_tokens_cache.find(start_key);
942
  if (it_def != global_def_tokens_cache.end()){
@@ -948,10 +943,10 @@ struct KnowledgeBase {
948
  }
949
 
950
  for (int depth = 1; depth < def_depth && !frontier.empty(); ++depth){
951
- std::vector<StrPtr> next_frontier;
952
 
953
  for (StrPtr w : frontier){
954
- const std::string key = normalize_dictionary_key(*w);
955
  if (key.empty()) continue;
956
 
957
  auto it2 = global_def_tokens_cache.find(key);
@@ -966,30 +961,30 @@ struct KnowledgeBase {
966
  frontier.swap(next_frontier);
967
  }
968
 
969
- std::vector<StrPtr> out;
970
  out.reserve(acc.size());
971
  for (StrPtr p : acc) out.push_back(p);
972
 
973
  {
974
- std::lock_guard<std::mutex> lk(def_m);
975
- def_index.emplace(wp, std::move(out));
976
  }
977
  }
978
 
979
- std::optional<NextSet> lookup_ngram(const std::vector<StrPtr>& ctx) const {
980
- std::lock_guard<std::mutex> lk(m);
981
  auto it = next.find(ctx);
982
- if (it == next.end()) return std::nullopt;
983
  return it->second;
984
  }
985
  };
986
 
987
- static void learn_tokens_ngram(KnowledgeBase &kb, const std::vector<std::string>& tokens, int n_gram_size) {
988
- std::vector<StrPtr> window;
989
  for (const auto& tok : tokens) {
990
  StrPtr tp = kb.interner.intern(tok);
991
  for (size_t j = 0; j < window.size(); ++j) {
992
- std::vector<StrPtr> ctx(window.begin() + j, window.end());
993
  kb.add_ngram(ctx, tp);
994
  }
995
  window.push_back(tp);
@@ -997,10 +992,10 @@ static void learn_tokens_ngram(KnowledgeBase &kb, const std::vector<std::string>
997
  }
998
  }
999
 
1000
- static std::vector<StrPtr>
1001
- intern_tokens(KnowledgeBase &kb, const std::vector<std::string> &tokens)
1002
  {
1003
- std::vector<StrPtr> out;
1004
  out.reserve(tokens.size());
1005
  for (const auto &t : tokens) out.push_back(kb.interner.intern(t));
1006
  return out;
@@ -1008,9 +1003,9 @@ intern_tokens(KnowledgeBase &kb, const std::vector<std::string> &tokens)
1008
 
1009
  static inline bool json_valid_index(size_t i, size_t n){ return i < n; }
1010
 
1011
- static std::string parse_quoted_string(const std::string &text, size_t &i){
1012
- std::string out;
1013
- if (!json_valid_index(i, text.size()) || text[i] != '"') throw std::runtime_error("expected '\"'");
1014
  ++i;
1015
  while (json_valid_index(i, text.size())){
1016
  char c = text[i++];
@@ -1026,14 +1021,14 @@ static std::string parse_quoted_string(const std::string &text, size_t &i){
1026
  return out;
1027
  }
1028
 
1029
- static void skip_spaces(const std::string &s, size_t &i){
1030
  while (json_valid_index(i, s.size()) && is_space(s[i])) ++i;
1031
  }
1032
 
1033
- static void skip_json_value(const std::string &s, size_t &i);
1034
 
1035
- static std::vector<std::string> parse_json_string_array(const std::string &text, size_t &i){
1036
- std::vector<std::string> out;
1037
  if (!json_valid_index(i, text.size()) || text[i] != '[') return out;
1038
 
1039
  ++i;
@@ -1053,7 +1048,7 @@ static std::vector<std::string> parse_json_string_array(const std::string &text,
1053
  return out;
1054
  }
1055
 
1056
- static void skip_json_value(const std::string &s, size_t &i){
1057
  skip_spaces(s, i);
1058
  if (!json_valid_index(i, s.size())) return;
1059
 
@@ -1104,11 +1099,11 @@ static void skip_json_value(const std::string &s, size_t &i){
1104
  }
1105
  }
1106
 
1107
- static std::vector<DictionaryEntry> parse_dictionary_json(){
1108
- std::vector<DictionaryEntry> dictionary;
1109
  if (dictionary_json_len == 0) return dictionary;
1110
 
1111
- std::string text;
1112
  text.reserve(dictionary_json_len);
1113
  for (unsigned int b = 0; b < dictionary_json_len; ++b){
1114
  text.push_back(static_cast<char>(dictionary_json[b]));
@@ -1139,7 +1134,7 @@ static std::vector<DictionaryEntry> parse_dictionary_json(){
1139
  if (!json_valid_index(i, text.size())) break;
1140
  if (text[i] == '}'){ ++i; break; }
1141
 
1142
- std::string field = parse_quoted_string(text, i);
1143
  skip_spaces(text, i);
1144
  if (!json_valid_index(i, text.size()) || text[i] != ':') break;
1145
  ++i;
@@ -1160,7 +1155,7 @@ static std::vector<DictionaryEntry> parse_dictionary_json(){
1160
  if (json_valid_index(i, text.size()) && text[i] == '}'){ ++i; break; }
1161
  }
1162
 
1163
- if (!entry.word.empty()) dictionary.push_back(std::move(entry));
1164
 
1165
  skip_spaces(text, i);
1166
  if (json_valid_index(i, text.size()) && text[i] == ','){ ++i; continue; }
@@ -1170,22 +1165,22 @@ static std::vector<DictionaryEntry> parse_dictionary_json(){
1170
  return dictionary;
1171
  }
1172
 
1173
- static std::string best_candidate_by_similarity(
1174
  const NextSet &cands,
1175
- const std::vector<StrPtr> &prompt_ptrs,
1176
- const std::vector<StrPtr> &resp_ptrs,
1177
- const std::unordered_map<StrPtr, std::vector<StrPtr>, PtrHash, PtrEq> &def_index,
1178
  const StringInterner &interner,
1179
- const std::unordered_map<std::string,int> &recent_counts,
1180
  double repeat_penalty,
1181
- const std::string &context_tok)
1182
  {
1183
- if (cands.empty()) return std::string();
1184
  if (cands.size() == 1) return *cands[0];
1185
 
1186
- const std::size_t words = std::max<std::size_t>(1, (interner.size() + 63u) / 64u);
1187
 
1188
- std::vector<std::uint64_t> agg_words(words, 0ULL);
1189
 
1190
  auto add_token_and_defs = [&](StrPtr t){
1191
  if (!t) return;
@@ -1203,8 +1198,8 @@ static std::string best_candidate_by_similarity(
1203
  }
1204
 
1205
  // Tokenize the expanded dictionary string to process sub-tokens
1206
- std::vector<std::string> sub_tokens = tokenize_others(*t);
1207
- for (const std::string& sub_tok : sub_tokens) {
1208
 
1209
  // Use id_of to check if the sub-token exists without adding a new string to the pool
1210
  TokenId sub_tid = interner.id_of(sub_tok);
@@ -1232,14 +1227,14 @@ static std::string best_candidate_by_similarity(
1232
  for (StrPtr t : prompt_ptrs) add_token_and_defs(t);
1233
  for (StrPtr t : resp_ptrs) add_token_and_defs(t);
1234
 
1235
- const std::size_t agg_count = bitset_count(agg_words.data(), words);
1236
 
1237
- const std::size_t M = cands.size();
1238
- std::vector<std::uint64_t> cand_words(M * words, 0ULL);
1239
- std::vector<std::size_t> cand_counts(M, 0);
1240
 
1241
- for (std::size_t i = 0; i < M; ++i){
1242
- std::uint64_t *row = cand_words.data() + i * words;
1243
  const StrPtr cand = cands[i];
1244
 
1245
  // Tag the candidate word itself
@@ -1256,8 +1251,8 @@ static std::string best_candidate_by_similarity(
1256
  }
1257
 
1258
  // Tokenize the candidate to process sub-tokens (e.g., camelCase, hyphens)
1259
- std::vector<std::string> sub_tokens = tokenize_others(*cand);
1260
- for (const std::string& sub_tok : sub_tokens) {
1261
 
1262
  // Check if the sub-token exists in the knowledge base
1263
  TokenId sub_tid = interner.id_of(sub_tok);
@@ -1285,7 +1280,7 @@ static std::string best_candidate_by_similarity(
1285
  cand_counts[i] = bitset_count(row, words);
1286
  }
1287
 
1288
- std::vector<double> scores(M, 0.0);
1289
 
1290
  #if defined(_OPENMP) && defined(CHATIPC_ENABLE_OMP_TARGET)
1291
  const bool use_target = (omp_get_num_devices() > 0) && (M >= 256);
@@ -1294,24 +1289,24 @@ static std::string best_candidate_by_similarity(
1294
  #endif
1295
 
1296
  if (use_target){
1297
- std::uint64_t *agg_ptr = agg_words.data();
1298
- std::uint64_t *cand_ptr = cand_words.data();
1299
- std::size_t *count_ptr = cand_counts.data();
1300
  double *score_ptr = scores.data();
1301
- const std::size_t cand_words_total = cand_words.size();
1302
 
1303
  #pragma omp target data map(to: agg_ptr[0:words], cand_ptr[0:cand_words_total], count_ptr[0:M]) map(from: score_ptr[0:M])
1304
  {
1305
  #pragma omp target teams distribute parallel for
1306
  for (ptrdiff_t i = 0; i < static_cast<ptrdiff_t>(M); ++i){
1307
- const std::uint64_t *row = cand_ptr + static_cast<std::size_t>(i) * words;
1308
 
1309
- std::size_t inter = 0;
1310
- for (std::size_t w = 0; w < words; ++w){
1311
  inter += popcount_u64(agg_ptr[w] & row[w]);
1312
  }
1313
 
1314
- const std::size_t uni = agg_count + count_ptr[(size_t)i] - inter;
1315
  score_ptr[(size_t)i] = uni ? static_cast<double>(inter) / static_cast<double>(uni) : 0.0;
1316
  }
1317
  }
@@ -1320,21 +1315,21 @@ static std::string best_candidate_by_similarity(
1320
  #pragma omp parallel for schedule(static)
1321
  #endif
1322
  for (ptrdiff_t i = 0; i < static_cast<ptrdiff_t>(M); ++i){
1323
- const std::uint64_t *row = cand_words.data() + static_cast<std::size_t>(i) * words;
1324
 
1325
- const std::size_t inter = bitset_intersection_count(agg_words.data(), row, words);
1326
- const std::size_t uni = agg_count + cand_counts[(size_t)i] - inter;
1327
  scores[(size_t)i] = uni ? static_cast<double>(inter) / static_cast<double>(uni) : 0.0;
1328
  }
1329
  }
1330
 
1331
  double best = -1e9;
1332
- std::string best_tok;
1333
 
1334
- for (std::size_t i = 0; i < M; ++i){
1335
- const std::string &tok = *cands[i];
1336
- const std::string tok_key = normalize_dictionary_key(tok);
1337
- const std::string count_key = tok_key.empty() ? tok : tok_key;
1338
 
1339
  auto rc_it = recent_counts.find(count_key);
1340
  const int cnt = (rc_it == recent_counts.end()) ? 0 : rc_it->second;
@@ -1353,24 +1348,24 @@ static std::string best_candidate_by_similarity(
1353
  return best_tok;
1354
  }
1355
 
1356
- static std::vector<std::string> construct_response(KnowledgeBase &kb,
1357
- const std::vector<std::string> &prompt_toks,
1358
  size_t response_maxlen,
1359
  double repeat_penalty,
1360
  int n_gram_size,
1361
  bool allow_path_loops)
1362
  {
1363
- std::vector<std::string> resp;
1364
  if (prompt_toks.empty() || response_maxlen == 0) return resp;
1365
 
1366
  auto prompt_ptrs = intern_tokens(kb, prompt_toks);
1367
- std::vector<StrPtr> resp_ptrs;
1368
- std::unordered_map<std::string, int> recent_counts;
1369
 
1370
  // The context combines prompt and generated text
1371
- std::vector<StrPtr> context = prompt_ptrs;
1372
 
1373
- auto would_create_2_cycle = [&](const std::string &cand) -> bool {
1374
  if (resp.size() < 3) return false;
1375
  return normalize_dictionary_key(cand) == normalize_dictionary_key(resp[resp.size() - 2]) &&
1376
  normalize_dictionary_key(resp.back()) == normalize_dictionary_key(resp[resp.size() - 3]);
@@ -1379,11 +1374,11 @@ static std::vector<std::string> construct_response(KnowledgeBase &kb,
1379
  for (size_t step = 0; step < response_maxlen; ++step){
1380
  NextSet candidates;
1381
  bool found = false;
1382
- std::string context_tok;
1383
 
1384
- int current_n = std::min(static_cast<int>(context.size()), n_gram_size);
1385
  while (current_n > 0) {
1386
- std::vector<StrPtr> search_ctx(context.end() - current_n, context.end());
1387
  auto opt = kb.lookup_ngram(search_ctx);
1388
  if (opt && !opt->empty()) {
1389
  candidates = *opt;
@@ -1397,8 +1392,8 @@ static std::vector<std::string> construct_response(KnowledgeBase &kb,
1397
  if (!found || candidates.empty()) break;
1398
 
1399
  if (candidates.size() == 1){
1400
- std::string only = *candidates[0];
1401
- std::string only_key = normalize_dictionary_key(only);
1402
  if (!allow_path_loops && recent_counts[only_key.empty() ? only : only_key] > 0) break;
1403
 
1404
  resp.push_back(only);
@@ -1406,11 +1401,11 @@ static std::vector<std::string> construct_response(KnowledgeBase &kb,
1406
  resp_ptrs.push_back(ptr);
1407
  context.push_back(ptr);
1408
  recent_counts[only_key.empty() ? only : only_key] += 1;
1409
- std::cout << only << ' ' << std::flush;
1410
  continue;
1411
  }
1412
 
1413
- std::string chosen = best_candidate_by_similarity(
1414
  candidates, prompt_ptrs, resp_ptrs, kb.def_index, kb.interner,
1415
  recent_counts, repeat_penalty, context_tok
1416
  );
@@ -1423,249 +1418,249 @@ static std::vector<std::string> construct_response(KnowledgeBase &kb,
1423
  resp_ptrs.push_back(chosen_ptr);
1424
  context.push_back(chosen_ptr);
1425
 
1426
- std::string chosen_key = normalize_dictionary_key(chosen);
1427
  recent_counts[chosen_key.empty() ? chosen : chosen_key] += 1;
1428
 
1429
- std::cout << chosen << ' ' << std::flush;
1430
  }
1431
 
1432
  return resp;
1433
  }
1434
 
1435
- static void learn_from_file(KnowledgeBase &kb, const std::string &fname, int n_gram_size){
1436
- std::ifstream ifs(fname);
1437
  if (!ifs) return;
1438
- std::vector<std::string> tokens;
1439
- std::string tok;
1440
  while (ifs >> tok) tokens.push_back(tok);
1441
  learn_tokens_ngram(kb, tokens, n_gram_size);
1442
  }
1443
 
1444
- static void learn_files_parallel(KnowledgeBase &kb, const std::vector<std::string> &files, int n_gram_size){
1445
  #pragma omp parallel for schedule(dynamic)
1446
  for (ptrdiff_t i=0;i<static_cast<ptrdiff_t>(files.size());++i){
1447
  learn_from_file(kb, files[(size_t)i], n_gram_size);
1448
  }
1449
  }
1450
 
1451
- static constexpr std::uint64_t KB_MAGIC = 0x434850434B535641ULL;
1452
- static constexpr std::uint64_t KB_VERSION = 2ULL;
1453
 
1454
- static void write_u64(std::ostream &os, std::uint64_t v) {
1455
  do {
1456
  unsigned char byte = v & 0x7F;
1457
  v >>= 7;
1458
  if (v != 0) byte |= 0x80;
1459
  os.put(static_cast<char>(byte));
1460
  } while (v != 0);
1461
- if (!os) throw std::runtime_error("write_u64 failed");
1462
  }
1463
 
1464
- static std::uint64_t read_u64(std::istream &is) {
1465
- std::uint64_t v = 0;
1466
  unsigned int shift = 0;
1467
  unsigned char byte;
1468
  do {
1469
  if (!is.read(reinterpret_cast<char*>(&byte), 1)) {
1470
- throw std::runtime_error("read_u64 failed");
1471
  }
1472
- v |= (static_cast<std::uint64_t>(byte & 0x7F) << shift);
1473
  shift += 7;
1474
  } while (byte & 0x80);
1475
  return v;
1476
  }
1477
 
1478
- static void write_string(std::ostream &os, const std::string &s){
1479
- write_u64(os, static_cast<std::uint64_t>(s.size()));
1480
  if (!s.empty()){
1481
- os.write(s.data(), static_cast<std::streamsize>(s.size()));
1482
- if(!os) throw std::runtime_error("write_string failed");
1483
  }
1484
  }
1485
 
1486
- static std::string read_string(std::istream &is){
1487
- std::uint64_t n = read_u64(is);
1488
- //if (n > (1ULL << 30)) throw std::runtime_error("save file is corrupted: string too large");
1489
 
1490
- std::string s;
1491
  s.resize(static_cast<size_t>(n));
1492
 
1493
  if (n != 0){
1494
- is.read(&s[0], static_cast<std::streamsize>(n));
1495
- if(!is) throw std::runtime_error("read_string failed");
1496
  }
1497
  return s;
1498
  }
1499
 
1500
- static void save_kb_binary(const KnowledgeBase &kb, const std::string &fname){
1501
- const std::string temp = fname + ".tmp";
1502
 
1503
  {
1504
- std::ofstream ofs(temp.c_str(), std::ios::binary | std::ios::trunc);
1505
- if (!ofs) throw std::runtime_error("cannot open temp save file");
1506
 
1507
- std::vector<std::string> pool;
1508
  pool.reserve(kb.interner.pool.size());
1509
  for (const auto &s : kb.interner.pool) pool.push_back(s);
1510
 
1511
- std::sort(pool.begin(), pool.end());
1512
 
1513
- std::unordered_map<std::string, std::uint64_t> id;
1514
  id.reserve(pool.size());
1515
- for (std::uint64_t i = 0; i < static_cast<std::uint64_t>(pool.size()); ++i){
1516
  id.emplace(pool[(size_t)i], i);
1517
  }
1518
 
1519
  write_u64(ofs, KB_MAGIC);
1520
  write_u64(ofs, KB_VERSION);
1521
- write_u64(ofs, static_cast<std::uint64_t>(kb.def_depth));
1522
 
1523
- write_u64(ofs, static_cast<std::uint64_t>(pool.size()));
1524
  for (const auto &s : pool) write_string(ofs, s);
1525
 
1526
- write_u64(ofs, static_cast<std::uint64_t>(kb.next.size()));
1527
  for (const auto &pr : kb.next){
1528
- write_u64(ofs, static_cast<std::uint64_t>(pr.first.size()));
1529
  for (StrPtr ctx_tok : pr.first){
1530
  write_u64(ofs, id.at(*ctx_tok));
1531
  }
1532
- write_u64(ofs, static_cast<std::uint64_t>(pr.second.size()));
1533
  for (StrPtr nxt : pr.second){
1534
  write_u64(ofs, id.at(*nxt));
1535
  }
1536
  }
1537
 
1538
- write_u64(ofs, static_cast<std::uint64_t>(kb.def_index.size()));
1539
  for (const auto &pr : kb.def_index){
1540
  write_u64(ofs, id.at(*pr.first));
1541
- write_u64(ofs, static_cast<std::uint64_t>(pr.second.size()));
1542
  for (StrPtr tok : pr.second){
1543
  write_u64(ofs, id.at(*tok));
1544
  }
1545
  }
1546
 
1547
  ofs.flush();
1548
- if (!ofs) throw std::runtime_error("failed while writing temp save file");
1549
  }
1550
 
1551
- std::remove(fname.c_str());
1552
- if (std::rename(temp.c_str(), fname.c_str()) != 0){
1553
- std::remove(temp.c_str());
1554
- throw std::runtime_error("failed to commit save file");
1555
  }
1556
  }
1557
 
1558
- static void load_kb_binary(KnowledgeBase &kb, const std::string &fname, int cli_def_depth){
1559
- std::ifstream ifs(fname, std::ios::binary);
1560
- if (!ifs) throw std::runtime_error("cannot open load file");
1561
 
1562
- const std::uint64_t magic = read_u64(ifs);
1563
- if (magic != KB_MAGIC) throw std::runtime_error("bad save file magic");
1564
 
1565
- const std::uint64_t version = read_u64(ifs);
1566
- if (version != KB_VERSION) throw std::runtime_error("unsupported save file version");
1567
 
1568
- const std::uint64_t file_def_depth = read_u64(ifs);
1569
 
1570
- const std::uint64_t N = read_u64(ifs);
1571
- //if (N > (1ULL << 26)) throw std::runtime_error("save file is corrupted: pool too large");
1572
 
1573
- std::vector<std::string> strings;
1574
  strings.reserve(static_cast<size_t>(N));
1575
 
1576
- for (std::uint64_t i = 0; i < N; ++i){
1577
  strings.push_back(read_string(ifs));
1578
  }
1579
 
1580
  kb.interner.pool.clear();
1581
  kb.interner.pool.reserve(static_cast<size_t>(N));
1582
 
1583
- std::vector<StrPtr> ptrs;
1584
  ptrs.reserve(static_cast<size_t>(N));
1585
  for (const auto &s : strings){
1586
  ptrs.push_back(kb.interner.intern(s));
1587
  }
1588
 
1589
  // Rebuild next
1590
- const std::uint64_t E = read_u64(ifs);
1591
- //if (E > (1ULL << 26)) throw std::runtime_error("save file is corrupted: graph too large");
1592
 
1593
  {
1594
- std::lock_guard<std::mutex> lk(kb.m);
1595
  kb.next.clear();
1596
  kb.next.reserve(static_cast<size_t>(E));
1597
  }
1598
 
1599
- for (std::uint64_t i = 0; i < E; ++i){
1600
- const std::uint64_t ctx_size = read_u64(ifs);
1601
- //if (ctx_size > (1ULL << 16)) throw std::runtime_error("save file is corrupted: n-gram context too large");
1602
 
1603
- std::vector<StrPtr> ctx;
1604
  ctx.reserve(static_cast<size_t>(ctx_size));
1605
- for(std::uint64_t c = 0; c < ctx_size; ++c) {
1606
- const std::uint64_t c_idx = read_u64(ifs);
1607
- if (c_idx >= ptrs.size()) throw std::runtime_error("save file is corrupted: bad context key");
1608
  ctx.push_back(ptrs[(size_t)c_idx]);
1609
  }
1610
 
1611
- const std::uint64_t M = read_u64(ifs);
1612
- //if (M > (1ULL << 26)) throw std::runtime_error("save file is corrupted: graph degree too large");
1613
 
1614
  NextSet vec;
1615
  vec.reserve(static_cast<size_t>(M));
1616
 
1617
- for (std::uint64_t j = 0; j < M; ++j){
1618
- const std::uint64_t v_idx = read_u64(ifs);
1619
- if (v_idx >= ptrs.size()) throw std::runtime_error("save file is corrupted: bad graph value");
1620
  vec.push_back(ptrs[(size_t)v_idx]);
1621
  }
1622
 
1623
  {
1624
- std::lock_guard<std::mutex> lk(kb.m);
1625
- kb.next.emplace(std::move(ctx), std::move(vec));
1626
  }
1627
  }
1628
 
1629
  // Rebuild def_index from file
1630
- const std::uint64_t K = read_u64(ifs);
1631
- //if (K > (1ULL << 26)) throw std::runtime_error("save file is corrupted: def_index too large");
1632
 
1633
  {
1634
- std::lock_guard<std::mutex> lk(kb.def_m);
1635
  kb.def_index.clear();
1636
  kb.def_index.reserve(static_cast<size_t>(K));
1637
  kb.def_depth = static_cast<int>(file_def_depth);
1638
  }
1639
 
1640
- for (std::uint64_t i = 0; i < K; ++i){
1641
- const std::uint64_t key_idx = read_u64(ifs);
1642
- const std::uint64_t M = read_u64(ifs);
1643
 
1644
- if (key_idx >= ptrs.size()) throw std::runtime_error("save file is corrupted: bad def key");
1645
- //if (M > (1ULL << 26)) throw std::runtime_error("save file is corrupted: def list too large");
1646
 
1647
- std::vector<StrPtr> toks;
1648
  toks.reserve(static_cast<size_t>(M));
1649
 
1650
- for (std::uint64_t j = 0; j < M; ++j){
1651
- const std::uint64_t v_idx = read_u64(ifs);
1652
- if (v_idx >= ptrs.size()) throw std::runtime_error("save file is corrupted: bad def value");
1653
  toks.push_back(ptrs[(size_t)v_idx]);
1654
  }
1655
 
1656
  {
1657
- std::lock_guard<std::mutex> lk(kb.def_m);
1658
- kb.def_index.emplace(ptrs[(size_t)key_idx], std::move(toks));
1659
  }
1660
  }
1661
 
1662
  if (cli_def_depth != static_cast<int>(file_def_depth)){
1663
  kb.set_def_depth(cli_def_depth);
1664
 
1665
- std::vector<StrPtr> targets;
1666
  targets.reserve(ptrs.size() + kb.next.size() * 2);
1667
 
1668
- std::unordered_set<StrPtr, PtrHash, PtrEq> seen;
1669
  seen.reserve(ptrs.size() + kb.next.size() * 2);
1670
 
1671
  for (StrPtr p : ptrs){
@@ -1673,7 +1668,7 @@ static void load_kb_binary(KnowledgeBase &kb, const std::string &fname, int cli_
1673
  }
1674
 
1675
  {
1676
- std::lock_guard<std::mutex> lk(kb.m);
1677
  for (const auto &pr : kb.next){
1678
  for (StrPtr ctx_element : pr.first) {
1679
  if (seen.insert(ctx_element).second) {
@@ -1696,112 +1691,111 @@ static void load_kb_binary(KnowledgeBase &kb, const std::string &fname, int cli_
1696
  }
1697
  }
1698
 
1699
- static void print_kb_info(const std::string &fname) {
1700
- std::ifstream ifs(fname, std::ios::binary);
1701
  if (!ifs) {
1702
- std::cerr << "Error: cannot open knowledge-base file " << fname << "\n";
1703
  return;
1704
  }
1705
 
1706
- std::cout << "Reckoning knowledge-base " << fname << " info. ";
1707
 
1708
  try {
1709
- const std::uint64_t magic = read_u64(ifs);
1710
- if (magic != KB_MAGIC) throw std::runtime_error("bad save file magic");
1711
 
1712
- const std::uint64_t version = read_u64(ifs);
1713
- if (version != KB_VERSION) throw std::runtime_error("unsupported save file version");
1714
 
1715
- const std::uint64_t file_def_depth = read_u64(ifs);
1716
 
1717
- const std::uint64_t N = read_u64(ifs);
1718
- //if (N > (1ULL << 26)) throw std::runtime_error("save file is corrupted: pool too large");
1719
 
1720
  // Dynamically read and skip strings since lengths are now variable-length encoded
1721
- for (std::uint64_t i = 0; i < N; ++i){
1722
- std::uint64_t len = read_u64(ifs);
1723
- //if (len > (1ULL << 30)) throw std::runtime_error("save file is corrupted: string too large");
1724
- if (len > 0) ifs.seekg(static_cast<std::streamoff>(len), std::ios_base::cur);
1725
  }
1726
 
1727
- const std::uint64_t E = read_u64(ifs);
1728
- //if (E > (1ULL << 26)) throw std::runtime_error("save file is corrupted: graph too large");
1729
 
1730
- std::uint64_t max_ngram = 0;
1731
- for (std::uint64_t i = 0; i < E; ++i){
1732
- const std::uint64_t ctx_size = read_u64(ifs);
1733
- //if (ctx_size > (1ULL << 16)) throw std::runtime_error("save file is corrupted: n-gram context too large");
1734
 
1735
  if (ctx_size > max_ngram) max_ngram = ctx_size;
1736
 
1737
  // Sequentially read (and skip) context keys using LEB128 stream parsing
1738
- for(std::uint64_t c = 0; c < ctx_size; ++c) {
1739
  read_u64(ifs);
1740
  }
1741
 
1742
- const std::uint64_t M = read_u64(ifs);
1743
- //if (M > (1ULL << 26)) throw std::runtime_error("save file is corrupted: graph degree too large");
1744
 
1745
  // Sequentially read (and skip) next-node values
1746
- for(std::uint64_t j = 0; j < M; ++j){
1747
  read_u64(ifs);
1748
  }
1749
  }
1750
 
1751
- std::cout << "The " << fname << " dictionary depth is " << file_def_depth << " and " << fname << " n-gram max size is " << max_ngram << "\n";
1752
 
1753
- } catch (const std::exception &e) {
1754
- std::cerr << "Error reading knowledge-base info: " << e.what() << "\n";
1755
  }
1756
  }
1757
 
1758
  static void print_commands(const char *p){
1759
- std::cout << p << " command-line interface options:\n";
1760
- std::cout << " --response-max-length N Set maximum number of token(s) in a response. Default number is 500.\n";
1761
- std::cout << " --save-kb FILE Save the knowledge-base graph to a binary file.\n";
1762
- std::cout << " --load-kb FILE Load a previously saved knowledge-base graph from a binary file.\n";
1763
- std::cout << " --info-kb FILE Show the dictionary-depth and n-gram max size of a knowledge-base file.\n";
1764
- std::cout << " --static-kb Disable learning prompt(s) and response(s) during interactive session.\n";
1765
- std::cout << " --dictionary-depth D Set depth of definition expansion used during learning. Default depth is 3.\n";
1766
- std::cout << " --n-gram-max-size N Set maximum size of the n-gram where N is the size. Default size is 3.\n";
1767
- std::cout << " --learn f1 f2 ... Learn from text file(s) to update the knowledge-base graph.\n";
1768
- std::cout << " --repeat-penalty P Set penalty for repeated tokens in response. Default penalty is 0.7 (higher penalty reduce repetition).\n";
1769
- std::cout << " --allow-path-loops Allow infinite loops if the knowledge-base graph has a circular single-path reference.\n";
1770
- std::cout << " --activate-agi Activate the Artificial General Intelligence (AGI) features.\n";
1771
- std::cout << " --help Show " << p << " command-line interface options.\n";
1772
  }
1773
 
1774
  int main(int argc, char **argv){
1775
  size_t response_maxlen = 500;
1776
- std::string savefile;
1777
- std::string load_txt;
1778
- std::string load_kb;
1779
  int def_depth = 3;
1780
  int n_gram_size = 3;
1781
  double repeat_penalty = 0.7; // default λ
1782
- std::vector<std::string> learn_files;
1783
  bool learn_chat = true;
1784
  bool allow_path_loops = false;
1785
 
1786
  for (int i=1;i<argc;++i){
1787
- std::string a = argv[i];
1788
  if (a=="--help"){ print_commands(argv[0]); return 0; }
1789
- if (a=="--response-max-length" && i+1<argc){ response_maxlen = std::stoul(argv[++i]); continue; }
1790
  if (a=="--save-kb" && i+1<argc){ savefile = argv[++i]; continue; }
1791
  if (a=="--load-kb" && i+1<argc){ load_kb = argv[++i]; continue; }
1792
  if (a=="--info-kb" && i+1<argc){ print_kb_info(argv[++i]); return 0; }
1793
  if (a=="--static-kb"){ learn_chat = false; continue; }
1794
- if (a=="--dictionary-depth" && i+1<argc){ def_depth = std::stoi(argv[++i]); continue; }
1795
- if (a=="--n-gram-max-size" && i+1<argc){ n_gram_size = std::max(1, std::stoi(argv[++i])); continue; }
1796
  if (a=="--learn"){ while(i+1<argc && argv[i+1][0] != '-') learn_files.push_back(argv[++i]); continue; }
1797
- if (a=="--repeat-penalty" && i+1<argc){ repeat_penalty = std::stod(argv[++i]); continue; }
1798
  if (a=="--allow-path-loops") { allow_path_loops = true; continue; }
1799
  if (a=="--activate-agi"){
1800
- std::cerr << "AGI subscription fee is $1,000 CAD per month. To pay the fee, contact Caleb Nwokocha via email cnwokocha@proton.me\n";
1801
  return 0;
1802
  }
1803
 
1804
- std::cerr << "Error: Unknown or incomplete command-line option '" << a << "'\n";
1805
  print_commands(argv[0]);
1806
  return 1;
1807
  }
@@ -1813,29 +1807,29 @@ int main(int argc, char **argv){
1813
  kb.set_def_depth(def_depth);
1814
 
1815
  if (!load_kb.empty()){
1816
- try { std::cerr << "Loading knowledge base: " << load_kb << "\n";
1817
- load_kb_binary(kb, load_kb, def_depth); std::cerr << "Loaded knowledge base: " << load_kb << "\n"; }
1818
- catch (const std::exception &e){ std::cerr << "Error: " << e.what() << "\n"; }
1819
  }
1820
 
1821
  if (!learn_files.empty()){
1822
- std::cerr << "Learning from file/s (" << learn_files.size() << ") using threads=" << omp_get_max_threads() << "\n";
1823
  learn_files_parallel(kb, learn_files, n_gram_size);
1824
  }
1825
 
1826
- std::string line;
1827
- std::cout << "Ready. Enter prompts.\n";
1828
- while (std::cout << "> " , std::getline(std::cin, line)){
1829
- if (line.empty()){ std::cout << "\n"; continue; }
1830
  auto prompt_toks = tokenize_whitespace(line);
1831
  if (learn_chat) {learn_tokens_ngram(kb, prompt_toks, n_gram_size);}
1832
  auto resp = construct_response(kb, prompt_toks, response_maxlen, repeat_penalty, n_gram_size, allow_path_loops);
1833
- std::cout << "\n";
1834
  if (!resp.empty() && learn_chat){learn_tokens_ngram(kb, resp, n_gram_size);}
1835
  if (!savefile.empty()){
1836
- try { std::cerr << "Saving knowledge base: " << savefile << "\n";
1837
- save_kb_binary(kb, savefile); std::cerr << "Saved knowledge base: " << savefile << "\n"; }
1838
- catch (const std::exception &e){ std::cerr << "Error: " << e.what() << "\n"; }
1839
  }
1840
  }
1841
 
 
1
  #include <algorithm>
 
2
  #include <cctype>
3
+ #include <cstdint>
4
+ #include <cstdio>
5
  #include <fstream>
6
  #include <iostream>
7
  #include <iterator>
 
8
  #include <mutex>
9
  #include <optional>
10
  #include <sstream>
11
  #include <stdexcept>
12
  #include <string>
 
13
  #include <unordered_map>
14
  #include <unordered_set>
15
  #include <vector>
 
18
  #include <omp.h>
19
  #else
20
  inline int omp_get_max_threads(){ return 1; }
 
21
  #endif
22
 
23
  extern unsigned char dictionary_json[];
24
  extern unsigned int dictionary_json_len;
25
 
26
+ using namespace std;
27
 
28
+ static inline bool is_space(char c){ return isspace(static_cast<unsigned char>(c)) != 0; }
29
+ static inline char to_low(char c){ return static_cast<char>(tolower(static_cast<unsigned char>(c))); }
 
30
 
31
  struct DictionaryEntry {
32
+ string pos;
33
+ string word;
34
+ vector<string> definitions;
35
  };
36
 
37
+ static vector<DictionaryEntry> global_dictionary_entries;
38
+ static unordered_map<string, vector<string>> global_def_tokens_cache;
39
+ static unordered_map<string, vector<string>> global_pos_cache;
40
 
41
  static inline bool is_word_char_for_key(char c){
42
  unsigned char uc = static_cast<unsigned char>(c);
43
+ return isalnum(uc) != 0 || c == '\'' || c == '-';
44
  }
45
 
46
+ static string normalize_dictionary_key(const string &s){
47
  size_t b = 0, e = s.size();
48
  while (b < e && !is_word_char_for_key(s[b])) ++b;
49
  while (e > b && !is_word_char_for_key(s[e - 1])) --e;
50
 
51
+ string out;
52
  out.reserve(e - b);
53
  for (size_t i = b; i < e; ++i) out.push_back(to_low(s[i]));
54
  return out;
55
  }
56
 
57
+ static string normalize_pos_tag(const string &s){
58
+ string out;
59
  out.reserve(s.size());
60
  for (char c : s){
61
  unsigned char uc = static_cast<unsigned char>(c);
62
+ if (isalpha(uc) != 0) out.push_back(to_low(c));
63
  }
64
  return out;
65
  }
 
78
  Interj
79
  };
80
 
81
+ static PosClass pos_class_from_tag(const string &tag){
82
  if (tag == "n" || tag == "noun") return PosClass::Noun;
83
  if (tag == "v" || tag == "verb" || tag == "part" || tag == "participle" || tag == "p") return PosClass::Verb;
84
  if (tag == "a" || tag == "adj" || tag == "adjective") return PosClass::Adj;
 
92
  return PosClass::Unknown;
93
  }
94
 
95
+ static bool has_pos_class(const vector<string> &tags, PosClass cls){
96
  for (const auto &t : tags){
97
  if (pos_class_from_tag(t) == cls) return true;
98
  }
99
  return false;
100
  }
101
 
102
+ static const vector<string> &dictionary_pos_for_token(const string &surface){
103
+ static const vector<string> empty;
104
  auto key = normalize_dictionary_key(surface);
105
  if (key.empty()) return empty;
106
 
 
108
  return (it == global_pos_cache.end()) ? empty : it->second;
109
  }
110
 
111
+ static bool first_alpha_is_upper(const string &s){
112
  for (char c : s){
113
  unsigned char uc = static_cast<unsigned char>(c);
114
+ if (isalpha(uc) != 0) return isupper(uc) != 0;
115
  }
116
  return false;
117
  }
118
 
119
+ static bool first_alpha_is_lower(const string &s){
120
  for (char c : s){
121
  unsigned char uc = static_cast<unsigned char>(c);
122
+ if (isalpha(uc) != 0) return islower(uc) != 0;
123
  }
124
  return false;
125
  }
126
 
127
+ static bool is_sentence_boundary_token(const string &s){
128
  if (s.empty()) return false;
129
  char c = s.back();
130
  return c == '.' || c == '!' || c == '?';
131
  }
132
 
133
+ static bool is_open_punct_token(const string &s){
134
  return s == "(" || s == "[" || s == "{" || s == "\"" || s == "'";
135
  }
136
 
137
+ static bool is_punctuation_only_token(const string &s){
138
  if (s.empty()) return false;
139
  for (char c : s){
140
  unsigned char uc = static_cast<unsigned char>(c);
141
+ if (isalnum(uc) != 0) return false;
142
  }
143
  return true;
144
  }
145
 
146
+ static string to_lower_ascii(string s) {
147
+ transform(s.begin(), s.end(), s.begin(),
148
+ [](unsigned char c) { return static_cast<char>(tolower(c)); });
149
  return s;
150
  }
151
 
152
+ static bool is_common_determiner(const string &s) {
153
+ static const unordered_set<string> words = {
154
  "a", "an", "the", "this", "that", "these", "those",
155
  "my", "your", "his", "her", "its", "our", "their",
156
  "all", "some", "any", "each", "every", "either", "neither",
 
163
  return words.find(to_lower_ascii(s)) != words.end();
164
  }
165
 
166
+ static bool is_common_preposition(const string &s) {
167
+ static const unordered_set<string> words = {
168
  "of", "in", "on", "at", "by", "for", "from", "with",
169
  "into", "onto", "about", "over", "under", "after", "before",
170
  "between", "through", "during", "without", "within", "across",
 
180
  return words.find(to_lower_ascii(s)) != words.end();
181
  }
182
 
183
+ static bool is_common_aux_or_modal(const string &s) {
184
+ static const unordered_set<string> words = {
185
  "to", "be", "am", "is", "are", "was", "were", "been", "being",
186
  "have", "has", "had", "do", "does", "did",
187
  "can", "could", "may", "might", "must", "shall", "should", "will",
 
193
  return words.find(to_lower_ascii(s)) != words.end();
194
  }
195
 
196
+ static bool begins_with_vowel_sound(const string &s){
197
  if (s.empty()) return false;
198
 
199
+ const string t = to_lower_ascii(s);
200
 
201
+ static const unordered_set<string> vowel_sound_exceptions = {
202
  "hour", "hours", "honest", "honesty", "honor", "honour", "honorable", "honourable",
203
  "heir", "heiress", "herb", "herbal", "herbalist", "homage", "hono(u)rary"
204
  };
205
 
206
+ static const unordered_set<string> consonant_sound_prefixes = {
207
  "uni", "univ", "unit", "use", "user", "euro", "eu", "one", "once", "ouija",
208
  "ufo", "ut", "uk", "ubiq", "ewe", "eul", "eup", "x", "y"
209
  };
 
220
  }
221
  }
222
 
 
223
  char c = t[0];
224
  return c == 'a' || c == 'e' || c == 'i' || c == 'o' || c == 'u';
225
  }
226
 
227
+ static double english_rule_bonus(const string &context_tok, const string &cand){
228
+ const string ctx_key = normalize_dictionary_key(context_tok);
229
+ const string cand_key = normalize_dictionary_key(cand);
230
 
231
  const auto &ctx_tags = dictionary_pos_for_token(context_tok);
232
  const auto &cand_tags = dictionary_pos_for_token(cand);
 
287
 
288
  if (is_punctuation_only_token(cand)){
289
  if (sentence_start) bonus -= 0.05;
290
+ else if (!context_tok.empty() && isalnum(static_cast<unsigned char>(context_tok.back())) != 0) bonus += 0.03;
291
  }
292
 
293
  if (is_sentence_boundary_token(cand)) bonus += 0.06;
 
295
  return bonus;
296
  }
297
 
298
+ static vector<string> tokenize_whitespace(const string &s){
299
+ istringstream iss(s);
300
+ vector<string> out;
301
+ string t;
302
  while (iss >> t) out.push_back(t);
303
  return out;
304
  }
305
 
306
  // Static helper to check dictionary presence using the existing pos cache
307
+ static inline bool check_dictionary(const string &word) {
308
  return !dictionary_pos_for_token(word).empty();
309
  }
310
 
311
  // Morphological lemmatizer to convert sub-tokens to their dictionary-defined base form.
312
+ // Returns optional<string>: contains the valid base form if found in the dictionary,
313
+ // or nullopt if the token has no definition and should be ignored.
314
+ static optional<string> get_valid_base_form(const string &word) {
315
+ if (word.empty()) return nullopt;
316
 
317
  // 1. BEFORE extracting/stemming, check if the original form is already in the dictionary
318
  if (check_dictionary(word)) {
 
322
  // 2. Continuous verb form (-ing)
323
  if (word.length() > 3 && word.substr(word.length() - 3) == "ing") {
324
  // e.g., playing -> play
325
+ string base1 = word.substr(0, word.length() - 3);
326
  if (check_dictionary(base1)) return base1;
327
 
328
  // e.g., making -> make
329
+ string base2 = base1 + "e";
330
  if (check_dictionary(base2)) return base2;
331
 
332
  // e.g., running -> run (doubled consonant check)
333
  if (base1.length() >= 2 && base1.back() == base1[base1.length() - 2]) {
334
+ string base3 = base1.substr(0, base1.length() - 1);
335
  if (check_dictionary(base3)) return base3;
336
  }
337
  }
 
339
  // 3. Past tense / Participle form (-ed)
340
  if (word.length() > 2 && word.substr(word.length() - 2) == "ed") {
341
  // e.g., baked -> bake
342
+ string base1 = word.substr(0, word.length() - 1);
343
  if (check_dictionary(base1)) return base1;
344
 
345
  // e.g., played -> play
346
+ string base2 = word.substr(0, word.length() - 2);
347
  if (check_dictionary(base2)) return base2;
348
 
349
  // e.g., hopped -> hop (doubled consonant check)
350
  if (base2.length() >= 2 && base2.back() == base2[base2.length() - 2]) {
351
+ string base3 = base2.substr(0, base2.length() - 1);
352
  if (check_dictionary(base3)) return base3;
353
  }
354
  }
 
356
  // 4. Superlative form (-est)
357
  if (word.length() > 3 && word.substr(word.length() - 3) == "est") {
358
  // e.g., simplest -> simple (strip 'st')
359
+ string base1 = word.substr(0, word.length() - 2);
360
  if (check_dictionary(base1)) return base1;
361
 
362
  // e.g., fastest -> fast
363
+ string base2 = word.substr(0, word.length() - 3);
364
  if (check_dictionary(base2)) return base2;
365
 
366
  // e.g., happiest -> happy (-iest -> -y)
367
  if (word.length() > 4 && word.substr(word.length() - 4) == "iest") {
368
+ string base_y = word.substr(0, word.length() - 4) + "y";
369
  if (check_dictionary(base_y)) return base_y;
370
  }
371
 
372
  // e.g., biggest -> big (doubled consonant check)
373
  if (base2.length() >= 2 && base2.back() == base2[base2.length() - 2]) {
374
+ string base3 = base2.substr(0, base2.length() - 1);
375
  if (check_dictionary(base3)) return base3;
376
  }
377
  }
 
379
  // 5. Comparative form (-er)
380
  if (word.length() > 2 && word.substr(word.length() - 2) == "er") {
381
  // e.g., simpler -> simple (strip 'r')
382
+ string base1 = word.substr(0, word.length() - 1);
383
  if (check_dictionary(base1)) return base1;
384
 
385
  // e.g., faster -> fast
386
+ string base2 = word.substr(0, word.length() - 2);
387
  if (check_dictionary(base2)) return base2;
388
 
389
  // e.g., happier -> happy (-ier -> -y)
390
  if (word.length() > 3 && word.substr(word.length() - 3) == "ier") {
391
+ string base_y = word.substr(0, word.length() - 3) + "y";
392
  if (check_dictionary(base_y)) return base_y;
393
  }
394
 
395
  // e.g., bigger -> big (doubled consonant check)
396
  if (base2.length() >= 2 && base2.back() == base2[base2.length() - 2]) {
397
+ string base3 = base2.substr(0, base2.length() - 1);
398
  if (check_dictionary(base3)) return base3;
399
  }
400
  }
 
402
  // 6. Adverbial form (-ly)
403
  if (word.length() > 2 && word.substr(word.length() - 2) == "ly") {
404
  // e.g., quickly -> quick
405
+ string base1 = word.substr(0, word.length() - 2);
406
  if (check_dictionary(base1)) return base1;
407
 
408
  // e.g., happily -> happy (-ily -> -y)
409
  if (word.length() > 3 && word.substr(word.length() - 3) == "ily") {
410
+ string base2 = word.substr(0, word.length() - 3) + "y";
411
  if (check_dictionary(base2)) return base2;
412
  }
413
 
414
  // e.g., basically -> basic (-ally -> -ic)
415
  if (word.length() > 4 && word.substr(word.length() - 4) == "ally") {
416
+ string base3 = word.substr(0, word.length() - 4);
417
  if (check_dictionary(base3)) return base3;
418
 
419
+ string base4 = base3 + "al";
420
  if (check_dictionary(base4)) return base4;
421
  }
422
 
423
  // e.g., gently -> gentle (-ly -> -le)
424
  if (word.back() == 'y' && word.length() > 2 && word[word.length() - 2] == 'l') {
425
+ string base5 = word.substr(0, word.length() - 1) + "e";
426
  if (check_dictionary(base5)) return base5;
427
  }
428
  }
 
430
  // 7. Noun suffix (-ness)
431
  if (word.length() > 4 && word.substr(word.length() - 4) == "ness") {
432
  // e.g., sadness -> sad
433
+ string base1 = word.substr(0, word.length() - 4);
434
  if (check_dictionary(base1)) return base1;
435
 
436
  // e.g., happiness -> happy (-iness -> -y)
437
  if (base1.length() > 1 && base1.back() == 'i') {
438
+ string base2 = base1.substr(0, base1.length() - 1) + "y";
439
  if (check_dictionary(base2)) return base2;
440
  }
441
  }
 
443
  // 8. Noun suffix (-ment)
444
  if (word.length() > 4 && word.substr(word.length() - 4) == "ment") {
445
  // e.g., development -> develop
446
+ string base1 = word.substr(0, word.length() - 4);
447
  if (check_dictionary(base1)) return base1;
448
  }
449
 
450
  // 9. Plural / 3rd-person singular form (-ies -> -y)
451
  if (word.length() > 3 && word.substr(word.length() - 3) == "ies") {
452
+ string base = word.substr(0, word.length() - 3) + "y";
453
  if (check_dictionary(base)) return base;
454
  }
455
 
456
  // 10. Plural / 3rd-person singular form (-ves -> -f / -fe)
457
  if (word.length() > 3 && word.substr(word.length() - 3) == "ves") {
458
+ string base1 = word.substr(0, word.length() - 3) + "f";
459
  if (check_dictionary(base1)) return base1;
460
 
461
+ string base2 = word.substr(0, word.length() - 3) + "fe";
462
  if (check_dictionary(base2)) return base2;
463
  }
464
 
465
  // 11. Plural form (-es)
466
  if (word.length() > 2 && word.substr(word.length() - 2) == "es") {
467
+ string base = word.substr(0, word.length() - 2);
468
  if (check_dictionary(base)) return base;
469
  }
470
 
471
  // 12. Plural form (-s)
472
  if (word.length() > 1 && word.back() == 's' && word[word.length() - 2] != 's') {
473
+ string base = word.substr(0, word.length() - 1);
474
  if (check_dictionary(base)) return base;
475
  }
476
 
477
  // No dictionary-defined base form found
478
+ return nullopt;
479
  }
480
 
481
  // Helper arrays for number-to-words
 
489
  "quattuordecillion", "quindecillion", "sexdecillion", "septendecillion",
490
  "octodecillion", "novemdecillion", "vigintillion", "googol"};
491
 
492
+ static vector<string> digit_to_words(const string& input_s) {
493
+ vector<string> words;
494
  if (input_s.empty()) return words;
495
 
496
  // Split the number into integer and fractional parts
497
+ string int_part = input_s;
498
+ string frac_part = "";
499
  size_t dot_pos = input_s.find('.');
500
+ if (dot_pos != string::npos) {
501
  int_part = input_s.substr(0, dot_pos);
502
  frac_part = input_s.substr(dot_pos + 1);
503
  }
504
 
505
  // Clean the integer part by removing commas
506
+ string s;
507
  for (char c : int_part) {
508
+ if (isdigit(static_cast<unsigned char>(c))) {
509
  s.push_back(c);
510
  }
511
  }
 
515
  // If there are no integer digits, ensure there are fractional digits before parsing
516
  bool has_frac_digits = false;
517
  for (char c : frac_part) {
518
+ if (isdigit(static_cast<unsigned char>(c))) has_frac_digits = true;
519
  }
520
  if (!has_frac_digits) return words;
521
  words.push_back("zero");
522
  } else {
523
  size_t first_non_zero = s.find_first_not_of('0');
524
+ if (first_non_zero == string::npos) {
525
  words.push_back("zero");
526
  } else {
527
+ string num = s.substr(first_non_zero);
528
 
529
  // Fallback for extremely large numbers to prevent overflow
530
  if (num.length() > 69) {
 
534
  } else {
535
  int scale_idx = 0;
536
  while (!num.empty()) {
537
+ int chunk_len = min<int>(3, num.length());
538
+ string chunk_str = num.substr(num.length() - chunk_len);
539
  num = num.substr(0, num.length() - chunk_len);
540
 
541
+ int chunk = stoi(chunk_str);
542
  if (chunk > 0) {
543
+ vector<string> chunk_words;
544
  int h = chunk / 100;
545
  int remainder = chunk % 100;
546
 
 
576
  if (!frac_part.empty()) {
577
  bool has_frac_digits = false;
578
  for (char c : frac_part) {
579
+ if (isdigit(static_cast<unsigned char>(c))) has_frac_digits = true;
580
  }
581
  if (has_frac_digits) {
582
  words.push_back("point");
583
  for (char c : frac_part) {
584
+ if (isdigit(static_cast<unsigned char>(c))) {
585
  words.push_back(ones[c - '0']);
586
  }
587
  }
 
591
  return words;
592
  }
593
 
594
+ static vector<string> tokenize_others(const string &s) {
595
  if (check_dictionary(s)) {
596
+ string lower_s;
597
  lower_s.reserve(s.size());
598
  for (char c : s) {
599
  lower_s.push_back(to_low(c));
 
603
 
604
  // Check if the entire string is a formatted number (pure digits, commas, and dots)
605
  auto is_num_char = [](unsigned char c) {
606
+ return isdigit(c) != 0 || c == ',' || c == '.';
607
  };
608
+ bool is_all_digits = !s.empty() && all_of(s.begin(), s.end(), is_num_char) &&
609
+ any_of(s.begin(), s.end(), [](unsigned char c) { return isdigit(c) != 0; });
610
 
611
  if (is_all_digits) {
612
  return digit_to_words(s);
613
  }
614
 
615
+ vector<string> out;
616
+ string cur;
617
 
618
  auto flush = [&]() {
619
  if (!cur.empty()) {
620
  // Check numbers inside mixed tokens (e.g. "word1,234.56")
621
+ bool cur_all_digits = all_of(cur.begin(), cur.end(), [](unsigned char c) {
622
+ return isdigit(c) != 0 || c == ',' || c == '.';
623
  });
624
+ bool has_digits = any_of(cur.begin(), cur.end(), [](unsigned char c) {
625
+ return isdigit(c) != 0;
626
  });
627
 
628
  if (cur_all_digits && has_digits) {
 
643
  char ch = static_cast<char>(uc);
644
 
645
  // Check for Operator/Symbol extraction
646
+ if (ispunct(uc) || (isprint(uc) && !isalnum(uc) && !isspace(uc))) {
647
 
648
  // Bypass regular punctuation extraction inside a formatted number
649
  if ((ch == ',' || ch == '.') &&
650
  !cur.empty() &&
651
+ (isdigit(static_cast<unsigned char>(cur.back())) != 0 || cur.back() == ',' || cur.back() == '.') &&
652
+ (i + 1 < s.size()) && isdigit(static_cast<unsigned char>(s[i+1])) != 0) {
653
  cur.push_back(ch);
654
  continue;
655
  }
656
 
657
  flush();
658
+ string sym(1, ch);
659
 
660
  // Check for double-character operators
661
  if (i + 1 < s.size()) {
662
  unsigned char nxt_uc = static_cast<unsigned char>(s[i+1]);
663
+ if (ispunct(nxt_uc) || (isprint(nxt_uc) && !isalnum(nxt_uc) && !isspace(nxt_uc))) {
664
+ string sym2 = sym + static_cast<char>(nxt_uc);
665
  if (check_dictionary(sym2)) {
666
  out.push_back(sym2);
667
  ++i;
 
675
  continue;
676
  }
677
 
678
+ if (isspace(uc)) {
679
  flush();
680
  continue;
681
  }
682
 
683
+ const bool is_upper = isupper(uc) != 0;
684
  bool camel_boundary = false;
685
  if (is_upper && !cur.empty()) {
686
  unsigned char prev = static_cast<unsigned char>(s[i - 1]);
687
+ const bool prev_lower_or_digit = (islower(prev) != 0) || (isdigit(prev) != 0);
688
+ const bool prev_upper = isupper(prev) != 0;
689
+ const bool next_lower = (i + 1 < s.size()) && (islower(static_cast<unsigned char>(s[i + 1])) != 0);
690
  camel_boundary = prev_lower_or_digit || (prev_upper && next_lower);
691
  }
692
 
693
  bool digit_boundary = false;
694
  if (!cur.empty()) {
695
  unsigned char prev = static_cast<unsigned char>(s[i - 1]);
696
+ bool prev_digit = (isdigit(prev) != 0) || prev == ',' || prev == '.';
697
+ bool curr_digit = isdigit(uc) != 0;
698
  if (prev_digit != curr_digit) {
699
  digit_boundary = true;
700
  }
 
704
  flush();
705
  }
706
 
707
+ if (isalnum(uc) != 0) {
708
+ cur.push_back(static_cast<char>(tolower(uc)));
709
  } else {
710
  flush();
711
  }
 
714
  return out;
715
  }
716
 
717
+ using StrPtr = const string*;
718
+ using TokenId = uint32_t;
719
  static constexpr TokenId TOKEN_ID_INVALID = 0xFFFFFFFFu;
720
 
721
+ static inline size_t popcount_u64(uint64_t x){
722
  #if defined(_MSC_VER)
723
+ return static_cast<size_t>(__popcnt64(x));
724
  #else
725
+ return static_cast<size_t>(__builtin_popcountll(static_cast<unsigned long long>(x)));
726
  #endif
727
  }
728
 
729
  struct StringInterner {
730
+ unordered_set<string> pool;
731
+ unordered_map<string, TokenId> id_by_string;
732
+ vector<const string*> string_by_id;
733
+ mutable mutex m;
734
 
735
+ const string* intern(const string &s){
736
+ lock_guard<mutex> lk(m);
737
 
738
  auto [it, inserted] = pool.emplace(s);
739
  if (inserted){
 
745
  return &*it;
746
  }
747
 
748
+ TokenId id_of(const string &s) const {
749
+ lock_guard<mutex> lk(m);
750
 
751
  auto it = id_by_string.find(s);
752
  return (it == id_by_string.end()) ? TOKEN_ID_INVALID : it->second;
753
  }
754
 
755
+ TokenId id_of(const string *p) const {
756
  return p ? id_of(*p) : TOKEN_ID_INVALID;
757
  }
758
 
759
+ const string* ptr_from_id(TokenId id) const {
760
+ lock_guard<mutex> lk(m);
761
 
762
  return (id < string_by_id.size()) ? string_by_id[(size_t)id] : nullptr;
763
  }
764
 
765
+ size_t size() const {
766
+ lock_guard<mutex> lk(m);
767
  return string_by_id.size();
768
  }
769
  };
770
 
771
+ static inline void bitset_set(uint64_t *bits, size_t words, TokenId id){
772
  if (id == TOKEN_ID_INVALID) return;
773
+ const size_t idx = static_cast<size_t>(id >> 6);
774
  if (idx >= words) return;
775
  bits[idx] |= (1ULL << (id & 63u));
776
  }
777
 
778
+ static inline size_t bitset_count(const uint64_t *bits, size_t words){
779
+ size_t total = 0;
780
+ for (size_t i = 0; i < words; ++i) total += popcount_u64(bits[i]);
781
  return total;
782
  }
783
 
784
+ static inline size_t bitset_intersection_count(const uint64_t *a, const uint64_t *b, size_t words){
785
+ size_t total = 0;
786
+ for (size_t i = 0; i < words; ++i) total += popcount_u64(a[i] & b[i]);
787
  return total;
788
  }
789
 
 
795
 
796
  #pragma omp parallel
797
  {
798
+ unordered_map<string, vector<string>> local_def_cache;
799
+ unordered_map<string, vector<string>> local_pos_cache;
800
 
801
  #pragma omp for schedule(guided) nowait
802
  for (int i = 0; i < num_entries; ++i){
803
  const auto &entry = global_dictionary_entries[i];
804
 
805
+ vector<string> all_def_toks;
806
  for (const auto &def : entry.definitions){
807
  auto toks = tokenize_others(def);
808
  all_def_toks.insert(all_def_toks.end(), toks.begin(), toks.end());
809
  }
810
 
811
+ string pos = normalize_pos_tag(entry.pos);
812
 
813
+ vector<string> sub_words;
814
+ string current_sub;
815
  for (char c : entry.word) {
816
  if (c == ';' || c == ',') {
817
  if (!current_sub.empty()) {
 
825
  if (!current_sub.empty()) sub_words.push_back(current_sub);
826
 
827
  for (size_t j = 0; j < sub_words.size(); ++j) {
828
+ const string key = normalize_dictionary_key(sub_words[j]);
829
  if (key.empty()) continue;
830
 
831
  if (!pos.empty()) {
832
  if (j == sub_words.size() - 1) {
833
+ local_pos_cache[key].push_back(move(pos));
834
  } else {
835
  local_pos_cache[key].push_back(pos);
836
  }
 
844
  {
845
  for (auto &pr : local_def_cache) {
846
  auto &vec = global_def_tokens_cache[pr.first];
847
+ vec.insert(vec.end(), make_move_iterator(pr.second.begin()), make_move_iterator(pr.second.end()));
848
  }
849
  for (auto &pr : local_pos_cache) {
850
  auto &vec = global_pos_cache[pr.first];
851
+ vec.insert(vec.end(), make_move_iterator(pr.second.begin()), make_move_iterator(pr.second.end()));
852
  }
853
  }
854
  }
855
 
856
  // Extract keys to use OpenMP parallel-for deduplication over hash map structures
857
+ vector<string> def_keys;
858
  def_keys.reserve(global_def_tokens_cache.size());
859
  for (const auto &pr : global_def_tokens_cache) def_keys.push_back(pr.first);
860
 
861
  #pragma omp parallel for schedule(dynamic)
862
  for (ptrdiff_t i = 0; i < static_cast<ptrdiff_t>(def_keys.size()); ++i){
863
  auto &v = global_def_tokens_cache[def_keys[i]];
864
+ sort(v.begin(), v.end());
865
+ v.erase(unique(v.begin(), v.end()), v.end());
866
  }
867
 
868
+ vector<string> pos_keys;
869
  pos_keys.reserve(global_pos_cache.size());
870
  for (const auto &pr : global_pos_cache) pos_keys.push_back(pr.first);
871
 
872
  #pragma omp parallel for schedule(dynamic)
873
  for (ptrdiff_t i = 0; i < static_cast<ptrdiff_t>(pos_keys.size()); ++i){
874
  auto &v = global_pos_cache[pos_keys[i]];
875
+ sort(v.begin(), v.end());
876
+ v.erase(unique(v.begin(), v.end()), v.end());
877
  }
878
  }
879
 
880
+ struct PtrHash { size_t operator()(StrPtr p) const noexcept { return hash<StrPtr>()(p); } };
881
  struct PtrEq { bool operator()(StrPtr a, StrPtr b) const noexcept { return a == b; } };
882
 
883
+ using NextSet = vector<StrPtr>;
884
 
885
  struct NgramHash {
886
+ size_t operator()(const vector<StrPtr>& v) const noexcept {
887
+ size_t seed = v.size();
888
  for(auto& p : v) {
889
+ seed ^= hash<StrPtr>()(p) + 0x9e3779b9 + (seed << 6) + (seed >> 2);
890
  }
891
  return seed;
892
  }
 
895
  struct KnowledgeBase {
896
  StringInterner interner;
897
  // Modified to use a vector of StrPtr (N-gram) as the key
898
+ unordered_map<vector<StrPtr>, NextSet, NgramHash> next;
899
+ mutable mutex m;
900
 
901
+ unordered_map<StrPtr, vector<StrPtr>, PtrHash, PtrEq> def_index;
902
+ mutable mutex def_m;
903
  int def_depth = 0;
904
 
905
+ void add_ngram(const vector<StrPtr>& ctx, StrPtr v){
906
  for (auto p : ctx) ensure_def_for_interned(p);
907
  ensure_def_for_interned(v);
908
 
909
+ lock_guard<mutex> lk(m);
910
  auto &vec = next[ctx];
911
  for (auto p : vec) if (p == v) return;
912
  vec.push_back(v);
913
  }
914
 
915
  void set_def_depth(int D){
916
+ lock_guard<mutex> lk(def_m);
917
  if (D != def_depth){
918
  def_index.clear();
919
  def_depth = D;
 
924
  if (wp == nullptr || def_depth <= 0) return;
925
 
926
  {
927
+ lock_guard<mutex> lk(def_m);
928
  if (def_index.find(wp) != def_index.end()) return;
929
  }
930
 
931
+ unordered_set<StrPtr, PtrHash, PtrEq> acc;
932
+ vector<StrPtr> frontier;
933
 
934
+ const string start_key = normalize_dictionary_key(*wp);
935
  if (!start_key.empty()){
936
  auto it_def = global_def_tokens_cache.find(start_key);
937
  if (it_def != global_def_tokens_cache.end()){
 
943
  }
944
 
945
  for (int depth = 1; depth < def_depth && !frontier.empty(); ++depth){
946
+ vector<StrPtr> next_frontier;
947
 
948
  for (StrPtr w : frontier){
949
+ const string key = normalize_dictionary_key(*w);
950
  if (key.empty()) continue;
951
 
952
  auto it2 = global_def_tokens_cache.find(key);
 
961
  frontier.swap(next_frontier);
962
  }
963
 
964
+ vector<StrPtr> out;
965
  out.reserve(acc.size());
966
  for (StrPtr p : acc) out.push_back(p);
967
 
968
  {
969
+ lock_guard<mutex> lk(def_m);
970
+ def_index.emplace(wp, move(out));
971
  }
972
  }
973
 
974
+ optional<NextSet> lookup_ngram(const vector<StrPtr>& ctx) const {
975
+ lock_guard<mutex> lk(m);
976
  auto it = next.find(ctx);
977
+ if (it == next.end()) return nullopt;
978
  return it->second;
979
  }
980
  };
981
 
982
+ static void learn_tokens_ngram(KnowledgeBase &kb, const vector<string>& tokens, int n_gram_size) {
983
+ vector<StrPtr> window;
984
  for (const auto& tok : tokens) {
985
  StrPtr tp = kb.interner.intern(tok);
986
  for (size_t j = 0; j < window.size(); ++j) {
987
+ vector<StrPtr> ctx(window.begin() + j, window.end());
988
  kb.add_ngram(ctx, tp);
989
  }
990
  window.push_back(tp);
 
992
  }
993
  }
994
 
995
+ static vector<StrPtr>
996
+ intern_tokens(KnowledgeBase &kb, const vector<string> &tokens)
997
  {
998
+ vector<StrPtr> out;
999
  out.reserve(tokens.size());
1000
  for (const auto &t : tokens) out.push_back(kb.interner.intern(t));
1001
  return out;
 
1003
 
1004
  static inline bool json_valid_index(size_t i, size_t n){ return i < n; }
1005
 
1006
+ static string parse_quoted_string(const string &text, size_t &i){
1007
+ string out;
1008
+ if (!json_valid_index(i, text.size()) || text[i] != '"') throw runtime_error("expected '\"'");
1009
  ++i;
1010
  while (json_valid_index(i, text.size())){
1011
  char c = text[i++];
 
1021
  return out;
1022
  }
1023
 
1024
+ static void skip_spaces(const string &s, size_t &i){
1025
  while (json_valid_index(i, s.size()) && is_space(s[i])) ++i;
1026
  }
1027
 
1028
+ static void skip_json_value(const string &s, size_t &i);
1029
 
1030
+ static vector<string> parse_json_string_array(const string &text, size_t &i){
1031
+ vector<string> out;
1032
  if (!json_valid_index(i, text.size()) || text[i] != '[') return out;
1033
 
1034
  ++i;
 
1048
  return out;
1049
  }
1050
 
1051
+ static void skip_json_value(const string &s, size_t &i){
1052
  skip_spaces(s, i);
1053
  if (!json_valid_index(i, s.size())) return;
1054
 
 
1099
  }
1100
  }
1101
 
1102
+ static vector<DictionaryEntry> parse_dictionary_json(){
1103
+ vector<DictionaryEntry> dictionary;
1104
  if (dictionary_json_len == 0) return dictionary;
1105
 
1106
+ string text;
1107
  text.reserve(dictionary_json_len);
1108
  for (unsigned int b = 0; b < dictionary_json_len; ++b){
1109
  text.push_back(static_cast<char>(dictionary_json[b]));
 
1134
  if (!json_valid_index(i, text.size())) break;
1135
  if (text[i] == '}'){ ++i; break; }
1136
 
1137
+ string field = parse_quoted_string(text, i);
1138
  skip_spaces(text, i);
1139
  if (!json_valid_index(i, text.size()) || text[i] != ':') break;
1140
  ++i;
 
1155
  if (json_valid_index(i, text.size()) && text[i] == '}'){ ++i; break; }
1156
  }
1157
 
1158
+ if (!entry.word.empty()) dictionary.push_back(move(entry));
1159
 
1160
  skip_spaces(text, i);
1161
  if (json_valid_index(i, text.size()) && text[i] == ','){ ++i; continue; }
 
1165
  return dictionary;
1166
  }
1167
 
1168
+ static string best_candidate_by_similarity(
1169
  const NextSet &cands,
1170
+ const vector<StrPtr> &prompt_ptrs,
1171
+ const vector<StrPtr> &resp_ptrs,
1172
+ const unordered_map<StrPtr, vector<StrPtr>, PtrHash, PtrEq> &def_index,
1173
  const StringInterner &interner,
1174
+ const unordered_map<string,int> &recent_counts,
1175
  double repeat_penalty,
1176
+ const string &context_tok)
1177
  {
1178
+ if (cands.empty()) return string();
1179
  if (cands.size() == 1) return *cands[0];
1180
 
1181
+ const size_t words = max<size_t>(1, (interner.size() + 63u) / 64u);
1182
 
1183
+ vector<uint64_t> agg_words(words, 0ULL);
1184
 
1185
  auto add_token_and_defs = [&](StrPtr t){
1186
  if (!t) return;
 
1198
  }
1199
 
1200
  // Tokenize the expanded dictionary string to process sub-tokens
1201
+ vector<string> sub_tokens = tokenize_others(*t);
1202
+ for (const string& sub_tok : sub_tokens) {
1203
 
1204
  // Use id_of to check if the sub-token exists without adding a new string to the pool
1205
  TokenId sub_tid = interner.id_of(sub_tok);
 
1227
  for (StrPtr t : prompt_ptrs) add_token_and_defs(t);
1228
  for (StrPtr t : resp_ptrs) add_token_and_defs(t);
1229
 
1230
+ const size_t agg_count = bitset_count(agg_words.data(), words);
1231
 
1232
+ const size_t M = cands.size();
1233
+ vector<uint64_t> cand_words(M * words, 0ULL);
1234
+ vector<size_t> cand_counts(M, 0);
1235
 
1236
+ for (size_t i = 0; i < M; ++i){
1237
+ uint64_t *row = cand_words.data() + i * words;
1238
  const StrPtr cand = cands[i];
1239
 
1240
  // Tag the candidate word itself
 
1251
  }
1252
 
1253
  // Tokenize the candidate to process sub-tokens (e.g., camelCase, hyphens)
1254
+ vector<string> sub_tokens = tokenize_others(*cand);
1255
+ for (const string& sub_tok : sub_tokens) {
1256
 
1257
  // Check if the sub-token exists in the knowledge base
1258
  TokenId sub_tid = interner.id_of(sub_tok);
 
1280
  cand_counts[i] = bitset_count(row, words);
1281
  }
1282
 
1283
+ vector<double> scores(M, 0.0);
1284
 
1285
  #if defined(_OPENMP) && defined(CHATIPC_ENABLE_OMP_TARGET)
1286
  const bool use_target = (omp_get_num_devices() > 0) && (M >= 256);
 
1289
  #endif
1290
 
1291
  if (use_target){
1292
+ uint64_t *agg_ptr = agg_words.data();
1293
+ uint64_t *cand_ptr = cand_words.data();
1294
+ size_t *count_ptr = cand_counts.data();
1295
  double *score_ptr = scores.data();
1296
+ const size_t cand_words_total = cand_words.size();
1297
 
1298
  #pragma omp target data map(to: agg_ptr[0:words], cand_ptr[0:cand_words_total], count_ptr[0:M]) map(from: score_ptr[0:M])
1299
  {
1300
  #pragma omp target teams distribute parallel for
1301
  for (ptrdiff_t i = 0; i < static_cast<ptrdiff_t>(M); ++i){
1302
+ const uint64_t *row = cand_ptr + static_cast<size_t>(i) * words;
1303
 
1304
+ size_t inter = 0;
1305
+ for (size_t w = 0; w < words; ++w){
1306
  inter += popcount_u64(agg_ptr[w] & row[w]);
1307
  }
1308
 
1309
+ const size_t uni = agg_count + count_ptr[(size_t)i] - inter;
1310
  score_ptr[(size_t)i] = uni ? static_cast<double>(inter) / static_cast<double>(uni) : 0.0;
1311
  }
1312
  }
 
1315
  #pragma omp parallel for schedule(static)
1316
  #endif
1317
  for (ptrdiff_t i = 0; i < static_cast<ptrdiff_t>(M); ++i){
1318
+ const uint64_t *row = cand_words.data() + static_cast<size_t>(i) * words;
1319
 
1320
+ const size_t inter = bitset_intersection_count(agg_words.data(), row, words);
1321
+ const size_t uni = agg_count + cand_counts[(size_t)i] - inter;
1322
  scores[(size_t)i] = uni ? static_cast<double>(inter) / static_cast<double>(uni) : 0.0;
1323
  }
1324
  }
1325
 
1326
  double best = -1e9;
1327
+ string best_tok;
1328
 
1329
+ for (size_t i = 0; i < M; ++i){
1330
+ const string &tok = *cands[i];
1331
+ const string tok_key = normalize_dictionary_key(tok);
1332
+ const string count_key = tok_key.empty() ? tok : tok_key;
1333
 
1334
  auto rc_it = recent_counts.find(count_key);
1335
  const int cnt = (rc_it == recent_counts.end()) ? 0 : rc_it->second;
 
1348
  return best_tok;
1349
  }
1350
 
1351
+ static vector<string> construct_response(KnowledgeBase &kb,
1352
+ const vector<string> &prompt_toks,
1353
  size_t response_maxlen,
1354
  double repeat_penalty,
1355
  int n_gram_size,
1356
  bool allow_path_loops)
1357
  {
1358
+ vector<string> resp;
1359
  if (prompt_toks.empty() || response_maxlen == 0) return resp;
1360
 
1361
  auto prompt_ptrs = intern_tokens(kb, prompt_toks);
1362
+ vector<StrPtr> resp_ptrs;
1363
+ unordered_map<string, int> recent_counts;
1364
 
1365
  // The context combines prompt and generated text
1366
+ vector<StrPtr> context = prompt_ptrs;
1367
 
1368
+ auto would_create_2_cycle = [&](const string &cand) -> bool {
1369
  if (resp.size() < 3) return false;
1370
  return normalize_dictionary_key(cand) == normalize_dictionary_key(resp[resp.size() - 2]) &&
1371
  normalize_dictionary_key(resp.back()) == normalize_dictionary_key(resp[resp.size() - 3]);
 
1374
  for (size_t step = 0; step < response_maxlen; ++step){
1375
  NextSet candidates;
1376
  bool found = false;
1377
+ string context_tok;
1378
 
1379
+ int current_n = min(static_cast<int>(context.size()), n_gram_size);
1380
  while (current_n > 0) {
1381
+ vector<StrPtr> search_ctx(context.end() - current_n, context.end());
1382
  auto opt = kb.lookup_ngram(search_ctx);
1383
  if (opt && !opt->empty()) {
1384
  candidates = *opt;
 
1392
  if (!found || candidates.empty()) break;
1393
 
1394
  if (candidates.size() == 1){
1395
+ string only = *candidates[0];
1396
+ string only_key = normalize_dictionary_key(only);
1397
  if (!allow_path_loops && recent_counts[only_key.empty() ? only : only_key] > 0) break;
1398
 
1399
  resp.push_back(only);
 
1401
  resp_ptrs.push_back(ptr);
1402
  context.push_back(ptr);
1403
  recent_counts[only_key.empty() ? only : only_key] += 1;
1404
+ cout << only << ' ' << flush;
1405
  continue;
1406
  }
1407
 
1408
+ string chosen = best_candidate_by_similarity(
1409
  candidates, prompt_ptrs, resp_ptrs, kb.def_index, kb.interner,
1410
  recent_counts, repeat_penalty, context_tok
1411
  );
 
1418
  resp_ptrs.push_back(chosen_ptr);
1419
  context.push_back(chosen_ptr);
1420
 
1421
+ string chosen_key = normalize_dictionary_key(chosen);
1422
  recent_counts[chosen_key.empty() ? chosen : chosen_key] += 1;
1423
 
1424
+ cout << chosen << ' ' << flush;
1425
  }
1426
 
1427
  return resp;
1428
  }
1429
 
1430
+ static void learn_from_file(KnowledgeBase &kb, const string &fname, int n_gram_size){
1431
+ ifstream ifs(fname);
1432
  if (!ifs) return;
1433
+ vector<string> tokens;
1434
+ string tok;
1435
  while (ifs >> tok) tokens.push_back(tok);
1436
  learn_tokens_ngram(kb, tokens, n_gram_size);
1437
  }
1438
 
1439
+ static void learn_files_parallel(KnowledgeBase &kb, const vector<string> &files, int n_gram_size){
1440
  #pragma omp parallel for schedule(dynamic)
1441
  for (ptrdiff_t i=0;i<static_cast<ptrdiff_t>(files.size());++i){
1442
  learn_from_file(kb, files[(size_t)i], n_gram_size);
1443
  }
1444
  }
1445
 
1446
+ static constexpr uint64_t KB_MAGIC = 0x434850434B535641ULL;
1447
+ static constexpr uint64_t KB_VERSION = 2ULL;
1448
 
1449
+ static void write_u64(ostream &os, uint64_t v) {
1450
  do {
1451
  unsigned char byte = v & 0x7F;
1452
  v >>= 7;
1453
  if (v != 0) byte |= 0x80;
1454
  os.put(static_cast<char>(byte));
1455
  } while (v != 0);
1456
+ if (!os) throw runtime_error("write_u64 failed");
1457
  }
1458
 
1459
+ static uint64_t read_u64(istream &is) {
1460
+ uint64_t v = 0;
1461
  unsigned int shift = 0;
1462
  unsigned char byte;
1463
  do {
1464
  if (!is.read(reinterpret_cast<char*>(&byte), 1)) {
1465
+ throw runtime_error("read_u64 failed");
1466
  }
1467
+ v |= (static_cast<uint64_t>(byte & 0x7F) << shift);
1468
  shift += 7;
1469
  } while (byte & 0x80);
1470
  return v;
1471
  }
1472
 
1473
+ static void write_string(ostream &os, const string &s){
1474
+ write_u64(os, static_cast<uint64_t>(s.size()));
1475
  if (!s.empty()){
1476
+ os.write(s.data(), static_cast<streamsize>(s.size()));
1477
+ if(!os) throw runtime_error("write_string failed");
1478
  }
1479
  }
1480
 
1481
+ static string read_string(istream &is){
1482
+ uint64_t n = read_u64(is);
1483
+ //if (n > (1ULL << 30)) throw runtime_error("save file is corrupted: string too large");
1484
 
1485
+ string s;
1486
  s.resize(static_cast<size_t>(n));
1487
 
1488
  if (n != 0){
1489
+ is.read(&s[0], static_cast<streamsize>(n));
1490
+ if(!is) throw runtime_error("read_string failed");
1491
  }
1492
  return s;
1493
  }
1494
 
1495
+ static void save_kb_binary(const KnowledgeBase &kb, const string &fname){
1496
+ const string temp = fname + ".tmp";
1497
 
1498
  {
1499
+ ofstream ofs(temp.c_str(), ios::binary | ios::trunc);
1500
+ if (!ofs) throw runtime_error("cannot open temp save file");
1501
 
1502
+ vector<string> pool;
1503
  pool.reserve(kb.interner.pool.size());
1504
  for (const auto &s : kb.interner.pool) pool.push_back(s);
1505
 
1506
+ sort(pool.begin(), pool.end());
1507
 
1508
+ unordered_map<string, uint64_t> id;
1509
  id.reserve(pool.size());
1510
+ for (uint64_t i = 0; i < static_cast<uint64_t>(pool.size()); ++i){
1511
  id.emplace(pool[(size_t)i], i);
1512
  }
1513
 
1514
  write_u64(ofs, KB_MAGIC);
1515
  write_u64(ofs, KB_VERSION);
1516
+ write_u64(ofs, static_cast<uint64_t>(kb.def_depth));
1517
 
1518
+ write_u64(ofs, static_cast<uint64_t>(pool.size()));
1519
  for (const auto &s : pool) write_string(ofs, s);
1520
 
1521
+ write_u64(ofs, static_cast<uint64_t>(kb.next.size()));
1522
  for (const auto &pr : kb.next){
1523
+ write_u64(ofs, static_cast<uint64_t>(pr.first.size()));
1524
  for (StrPtr ctx_tok : pr.first){
1525
  write_u64(ofs, id.at(*ctx_tok));
1526
  }
1527
+ write_u64(ofs, static_cast<uint64_t>(pr.second.size()));
1528
  for (StrPtr nxt : pr.second){
1529
  write_u64(ofs, id.at(*nxt));
1530
  }
1531
  }
1532
 
1533
+ write_u64(ofs, static_cast<uint64_t>(kb.def_index.size()));
1534
  for (const auto &pr : kb.def_index){
1535
  write_u64(ofs, id.at(*pr.first));
1536
+ write_u64(ofs, static_cast<uint64_t>(pr.second.size()));
1537
  for (StrPtr tok : pr.second){
1538
  write_u64(ofs, id.at(*tok));
1539
  }
1540
  }
1541
 
1542
  ofs.flush();
1543
+ if (!ofs) throw runtime_error("failed while writing temp save file");
1544
  }
1545
 
1546
+ remove(fname.c_str());
1547
+ if (rename(temp.c_str(), fname.c_str()) != 0){
1548
+ remove(temp.c_str());
1549
+ throw runtime_error("failed to commit save file");
1550
  }
1551
  }
1552
 
1553
+ static void load_kb_binary(KnowledgeBase &kb, const string &fname, int cli_def_depth){
1554
+ ifstream ifs(fname, ios::binary);
1555
+ if (!ifs) throw runtime_error("cannot open load file");
1556
 
1557
+ const uint64_t magic = read_u64(ifs);
1558
+ if (magic != KB_MAGIC) throw runtime_error("bad save file magic");
1559
 
1560
+ const uint64_t version = read_u64(ifs);
1561
+ if (version != KB_VERSION) throw runtime_error("unsupported save file version");
1562
 
1563
+ const uint64_t file_def_depth = read_u64(ifs);
1564
 
1565
+ const uint64_t N = read_u64(ifs);
1566
+ //if (N > (1ULL << 26)) throw runtime_error("save file is corrupted: pool too large");
1567
 
1568
+ vector<string> strings;
1569
  strings.reserve(static_cast<size_t>(N));
1570
 
1571
+ for (uint64_t i = 0; i < N; ++i){
1572
  strings.push_back(read_string(ifs));
1573
  }
1574
 
1575
  kb.interner.pool.clear();
1576
  kb.interner.pool.reserve(static_cast<size_t>(N));
1577
 
1578
+ vector<StrPtr> ptrs;
1579
  ptrs.reserve(static_cast<size_t>(N));
1580
  for (const auto &s : strings){
1581
  ptrs.push_back(kb.interner.intern(s));
1582
  }
1583
 
1584
  // Rebuild next
1585
+ const uint64_t E = read_u64(ifs);
1586
+ //if (E > (1ULL << 26)) throw runtime_error("save file is corrupted: graph too large");
1587
 
1588
  {
1589
+ lock_guard<mutex> lk(kb.m);
1590
  kb.next.clear();
1591
  kb.next.reserve(static_cast<size_t>(E));
1592
  }
1593
 
1594
+ for (uint64_t i = 0; i < E; ++i){
1595
+ const uint64_t ctx_size = read_u64(ifs);
1596
+ //if (ctx_size > (1ULL << 16)) throw runtime_error("save file is corrupted: n-gram context too large");
1597
 
1598
+ vector<StrPtr> ctx;
1599
  ctx.reserve(static_cast<size_t>(ctx_size));
1600
+ for(uint64_t c = 0; c < ctx_size; ++c) {
1601
+ const uint64_t c_idx = read_u64(ifs);
1602
+ if (c_idx >= ptrs.size()) throw runtime_error("save file is corrupted: bad context key");
1603
  ctx.push_back(ptrs[(size_t)c_idx]);
1604
  }
1605
 
1606
+ const uint64_t M = read_u64(ifs);
1607
+ //if (M > (1ULL << 26)) throw runtime_error("save file is corrupted: graph degree too large");
1608
 
1609
  NextSet vec;
1610
  vec.reserve(static_cast<size_t>(M));
1611
 
1612
+ for (uint64_t j = 0; j < M; ++j){
1613
+ const uint64_t v_idx = read_u64(ifs);
1614
+ if (v_idx >= ptrs.size()) throw runtime_error("save file is corrupted: bad graph value");
1615
  vec.push_back(ptrs[(size_t)v_idx]);
1616
  }
1617
 
1618
  {
1619
+ lock_guard<mutex> lk(kb.m);
1620
+ kb.next.emplace(move(ctx), move(vec));
1621
  }
1622
  }
1623
 
1624
  // Rebuild def_index from file
1625
+ const uint64_t K = read_u64(ifs);
1626
+ //if (K > (1ULL << 26)) throw runtime_error("save file is corrupted: def_index too large");
1627
 
1628
  {
1629
+ lock_guard<mutex> lk(kb.def_m);
1630
  kb.def_index.clear();
1631
  kb.def_index.reserve(static_cast<size_t>(K));
1632
  kb.def_depth = static_cast<int>(file_def_depth);
1633
  }
1634
 
1635
+ for (uint64_t i = 0; i < K; ++i){
1636
+ const uint64_t key_idx = read_u64(ifs);
1637
+ const uint64_t M = read_u64(ifs);
1638
 
1639
+ if (key_idx >= ptrs.size()) throw runtime_error("save file is corrupted: bad def key");
1640
+ //if (M > (1ULL << 26)) throw runtime_error("save file is corrupted: def list too large");
1641
 
1642
+ vector<StrPtr> toks;
1643
  toks.reserve(static_cast<size_t>(M));
1644
 
1645
+ for (uint64_t j = 0; j < M; ++j){
1646
+ const uint64_t v_idx = read_u64(ifs);
1647
+ if (v_idx >= ptrs.size()) throw runtime_error("save file is corrupted: bad def value");
1648
  toks.push_back(ptrs[(size_t)v_idx]);
1649
  }
1650
 
1651
  {
1652
+ lock_guard<mutex> lk(kb.def_m);
1653
+ kb.def_index.emplace(ptrs[(size_t)key_idx], move(toks));
1654
  }
1655
  }
1656
 
1657
  if (cli_def_depth != static_cast<int>(file_def_depth)){
1658
  kb.set_def_depth(cli_def_depth);
1659
 
1660
+ vector<StrPtr> targets;
1661
  targets.reserve(ptrs.size() + kb.next.size() * 2);
1662
 
1663
+ unordered_set<StrPtr, PtrHash, PtrEq> seen;
1664
  seen.reserve(ptrs.size() + kb.next.size() * 2);
1665
 
1666
  for (StrPtr p : ptrs){
 
1668
  }
1669
 
1670
  {
1671
+ lock_guard<mutex> lk(kb.m);
1672
  for (const auto &pr : kb.next){
1673
  for (StrPtr ctx_element : pr.first) {
1674
  if (seen.insert(ctx_element).second) {
 
1691
  }
1692
  }
1693
 
1694
+ static void print_kb_info(const string &fname) {
1695
+ ifstream ifs(fname, ios::binary);
1696
  if (!ifs) {
1697
+ cerr << "Error: cannot open knowledge-base file " << fname << "\n";
1698
  return;
1699
  }
1700
 
1701
+ cout << "Reckoning knowledge-base " << fname << " info. ";
1702
 
1703
  try {
1704
+ const uint64_t magic = read_u64(ifs);
1705
+ if (magic != KB_MAGIC) throw runtime_error("bad save file magic");
1706
 
1707
+ const uint64_t version = read_u64(ifs);
1708
+ if (version != KB_VERSION) throw runtime_error("unsupported save file version");
1709
 
1710
+ const uint64_t file_def_depth = read_u64(ifs);
1711
 
1712
+ const uint64_t N = read_u64(ifs);
1713
+ //if (N > (1ULL << 26)) throw runtime_error("save file is corrupted: pool too large");
1714
 
1715
  // Dynamically read and skip strings since lengths are now variable-length encoded
1716
+ for (uint64_t i = 0; i < N; ++i){
1717
+ uint64_t len = read_u64(ifs);
1718
+ //if (len > (1ULL << 30)) throw runtime_error("save file is corrupted: string too large");
1719
+ if (len > 0) ifs.seekg(static_cast<streamoff>(len), ios_base::cur);
1720
  }
1721
 
1722
+ const uint64_t E = read_u64(ifs);
1723
+ //if (E > (1ULL << 26)) throw runtime_error("save file is corrupted: graph too large");
1724
 
1725
+ uint64_t max_ngram = 0;
1726
+ for (uint64_t i = 0; i < E; ++i){
1727
+ const uint64_t ctx_size = read_u64(ifs);
1728
+ //if (ctx_size > (1ULL << 16)) throw runtime_error("save file is corrupted: n-gram context too large");
1729
 
1730
  if (ctx_size > max_ngram) max_ngram = ctx_size;
1731
 
1732
  // Sequentially read (and skip) context keys using LEB128 stream parsing
1733
+ for(uint64_t c = 0; c < ctx_size; ++c) {
1734
  read_u64(ifs);
1735
  }
1736
 
1737
+ const uint64_t M = read_u64(ifs);
1738
+ //if (M > (1ULL << 26)) throw runtime_error("save file is corrupted: graph degree too large");
1739
 
1740
  // Sequentially read (and skip) next-node values
1741
+ for(uint64_t j = 0; j < M; ++j){
1742
  read_u64(ifs);
1743
  }
1744
  }
1745
 
1746
+ cout << "The " << fname << " dictionary depth is " << file_def_depth << " and " << fname << " n-gram max size is " << max_ngram << "\n";
1747
 
1748
+ } catch (const exception &e) {
1749
+ cerr << "Error reading knowledge-base info: " << e.what() << "\n";
1750
  }
1751
  }
1752
 
1753
  static void print_commands(const char *p){
1754
+ cout << p << " command-line interface options:\n";
1755
+ cout << " --response-max-length N Set maximum number of token(s) in a response. Default number is 500.\n";
1756
+ cout << " --save-kb FILE Save the knowledge-base graph to a binary file.\n";
1757
+ cout << " --load-kb FILE Load a previously saved knowledge-base graph from a binary file.\n";
1758
+ cout << " --info-kb FILE Show the dictionary-depth and n-gram max size of a knowledge-base file.\n";
1759
+ cout << " --static-kb Disable learning prompt(s) and response(s) during interactive session.\n";
1760
+ cout << " --dictionary-depth D Set depth of definition expansion used during learning. Default depth is 3.\n";
1761
+ cout << " --n-gram-max-size N Set maximum size of the n-gram where N is the size. Default size is 3.\n";
1762
+ cout << " --learn f1 f2 ... Learn from text file(s) to update the knowledge-base graph.\n";
1763
+ cout << " --repeat-penalty P Set penalty for repeated tokens in response. Default penalty is 0.7 (higher penalty reduce repetition).\n";
1764
+ cout << " --allow-path-loops Allow infinite loops if the knowledge-base graph has a circular single-path reference.\n";
1765
+ cout << " --activate-agi Activate the Artificial General Intelligence (AGI) features.\n";
1766
+ cout << " --help Show " << p << " command-line interface options.\n";
1767
  }
1768
 
1769
  int main(int argc, char **argv){
1770
  size_t response_maxlen = 500;
1771
+ string savefile;
1772
+ string load_kb;
 
1773
  int def_depth = 3;
1774
  int n_gram_size = 3;
1775
  double repeat_penalty = 0.7; // default λ
1776
+ vector<string> learn_files;
1777
  bool learn_chat = true;
1778
  bool allow_path_loops = false;
1779
 
1780
  for (int i=1;i<argc;++i){
1781
+ string a = argv[i];
1782
  if (a=="--help"){ print_commands(argv[0]); return 0; }
1783
+ if (a=="--response-max-length" && i+1<argc){ response_maxlen = stoul(argv[++i]); continue; }
1784
  if (a=="--save-kb" && i+1<argc){ savefile = argv[++i]; continue; }
1785
  if (a=="--load-kb" && i+1<argc){ load_kb = argv[++i]; continue; }
1786
  if (a=="--info-kb" && i+1<argc){ print_kb_info(argv[++i]); return 0; }
1787
  if (a=="--static-kb"){ learn_chat = false; continue; }
1788
+ if (a=="--dictionary-depth" && i+1<argc){ def_depth = stoi(argv[++i]); continue; }
1789
+ if (a=="--n-gram-max-size" && i+1<argc){ n_gram_size = max(1, stoi(argv[++i])); continue; }
1790
  if (a=="--learn"){ while(i+1<argc && argv[i+1][0] != '-') learn_files.push_back(argv[++i]); continue; }
1791
+ if (a=="--repeat-penalty" && i+1<argc){ repeat_penalty = stod(argv[++i]); continue; }
1792
  if (a=="--allow-path-loops") { allow_path_loops = true; continue; }
1793
  if (a=="--activate-agi"){
1794
+ cerr << "AGI subscription fee is $1,000 CAD per month. To pay the fee, contact Caleb Nwokocha via email cnwokocha@proton.me\n";
1795
  return 0;
1796
  }
1797
 
1798
+ cerr << "Error: Unknown or incomplete command-line option '" << a << "'\n";
1799
  print_commands(argv[0]);
1800
  return 1;
1801
  }
 
1807
  kb.set_def_depth(def_depth);
1808
 
1809
  if (!load_kb.empty()){
1810
+ try { cerr << "Loading knowledge base: " << load_kb << "\n";
1811
+ load_kb_binary(kb, load_kb, def_depth); cerr << "Loaded knowledge base: " << load_kb << "\n"; }
1812
+ catch (const exception &e){ cerr << "Error: " << e.what() << "\n"; }
1813
  }
1814
 
1815
  if (!learn_files.empty()){
1816
+ cerr << "Learning from file/s (" << learn_files.size() << ") using threads=" << omp_get_max_threads() << "\n";
1817
  learn_files_parallel(kb, learn_files, n_gram_size);
1818
  }
1819
 
1820
+ string line;
1821
+ cout << "Ready. Enter prompts.\n";
1822
+ while (cout << "> " , getline(cin, line)){
1823
+ if (line.empty()){ cout << "\n"; continue; }
1824
  auto prompt_toks = tokenize_whitespace(line);
1825
  if (learn_chat) {learn_tokens_ngram(kb, prompt_toks, n_gram_size);}
1826
  auto resp = construct_response(kb, prompt_toks, response_maxlen, repeat_penalty, n_gram_size, allow_path_loops);
1827
+ cout << "\n";
1828
  if (!resp.empty() && learn_chat){learn_tokens_ngram(kb, resp, n_gram_size);}
1829
  if (!savefile.empty()){
1830
+ try { cerr << "Saving knowledge base: " << savefile << "\n";
1831
+ save_kb_binary(kb, savefile); cerr << "Saved knowledge base: " << savefile << "\n"; }
1832
+ catch (const exception &e){ cerr << "Error: " << e.what() << "\n"; }
1833
  }
1834
  }
1835