| #include <algorithm>
|
| #include <cstdint>
|
| #include <cstdio>
|
| #include <fstream>
|
| #include <iostream>
|
| #include <mutex>
|
| #include <optional>
|
| #include <sstream>
|
| #include <stdexcept>
|
| #include <string>
|
| #include <unordered_map>
|
| #include <unordered_set>
|
| #include <vector>
|
|
|
| #ifdef _OPENMP
|
| #include <omp.h>
|
| #else
|
| inline int omp_get_max_threads(){ return 1; }
|
| #endif
|
|
|
| using namespace std;
|
|
|
| static vector<string> tokenize_whitespace(const string &s){
|
| istringstream iss(s);
|
| vector<string> out;
|
| string t;
|
| while (iss >> t) out.push_back(t);
|
| return out;
|
| }
|
|
|
| using StrPtr = const string*;
|
| using TokenId = uint32_t;
|
| static constexpr TokenId TOKEN_ID_INVALID = 0xFFFFFFFFu;
|
|
|
| static inline size_t popcount_u64(uint64_t x){
|
| #if defined(_MSC_VER)
|
| return static_cast<size_t>(__popcnt64(x));
|
| #else
|
| return static_cast<size_t>(__builtin_popcountll(static_cast<unsigned long long>(x)));
|
| #endif
|
| }
|
|
|
| struct StringInterner {
|
| unordered_set<string> pool;
|
| unordered_map<string, TokenId> id_by_string;
|
| vector<const string*> string_by_id;
|
| mutable mutex m;
|
|
|
| const string* intern(const string &s){
|
| lock_guard<mutex> lk(m);
|
|
|
| auto [it, inserted] = pool.emplace(s);
|
| if (inserted){
|
| const TokenId id = static_cast<TokenId>(string_by_id.size());
|
| id_by_string.emplace(*it, id);
|
| string_by_id.push_back(&*it);
|
| }
|
|
|
| return &*it;
|
| }
|
|
|
| TokenId id_of(const string &s) const {
|
| lock_guard<mutex> lk(m);
|
|
|
| auto it = id_by_string.find(s);
|
| return (it == id_by_string.end()) ? TOKEN_ID_INVALID : it->second;
|
| }
|
|
|
| TokenId id_of(const string *p) const {
|
| return p ? id_of(*p) : TOKEN_ID_INVALID;
|
| }
|
|
|
| size_t size() const {
|
| lock_guard<mutex> lk(m);
|
| return string_by_id.size();
|
| }
|
| };
|
|
|
| static inline void bitset_set(uint64_t *bits, size_t words, TokenId id){
|
| if (id == TOKEN_ID_INVALID) return;
|
| const size_t idx = static_cast<size_t>(id >> 6);
|
| if (idx >= words) return;
|
| bits[idx] |= (1ULL << (id & 63u));
|
| }
|
|
|
| static inline size_t bitset_count(const uint64_t *bits, size_t words){
|
| size_t total = 0;
|
| for (size_t i = 0; i < words; ++i) total += popcount_u64(bits[i]);
|
| return total;
|
| }
|
|
|
| using NextSet = vector<StrPtr>;
|
|
|
| struct NgramHash {
|
| size_t operator()(const vector<StrPtr>& v) const noexcept {
|
| size_t seed = v.size();
|
| for(auto& p : v) {
|
| seed ^= hash<StrPtr>()(p) + 0x9e3779b9 + (seed << 6) + (seed >> 2);
|
| }
|
| return seed;
|
| }
|
| };
|
|
|
| struct KnowledgeBase {
|
| StringInterner interner;
|
|
|
| unordered_map<vector<StrPtr>, NextSet, NgramHash> next;
|
| mutable mutex m;
|
|
|
| void add_ngram(const vector<StrPtr>& ctx, StrPtr v){
|
| lock_guard<mutex> lk(m);
|
| auto &vec = next[ctx];
|
| for (auto p : vec) if (p == v) return;
|
| vec.push_back(v);
|
| }
|
|
|
| optional<NextSet> lookup_ngram(const vector<StrPtr>& ctx) const {
|
| lock_guard<mutex> lk(m);
|
| auto it = next.find(ctx);
|
| if (it == next.end()) return nullopt;
|
| return it->second;
|
| }
|
|
|
|
|
| const NextSet* lookup_ngram_fast(const vector<StrPtr>& ctx) const {
|
| auto it = next.find(ctx);
|
| return (it != next.end()) ? &(it->second) : nullptr;
|
| }
|
| };
|
|
|
| static void learn_tokens_ngram(KnowledgeBase &kb, const vector<string>& tokens, int n_gram_size) {
|
| vector<StrPtr> window;
|
| for (const auto& tok : tokens) {
|
| StrPtr tp = kb.interner.intern(tok);
|
| for (size_t j = 0; j < window.size(); ++j) {
|
| vector<StrPtr> ctx(window.begin() + j, window.end());
|
| kb.add_ngram(ctx, tp);
|
| }
|
| window.push_back(tp);
|
| if (window.size() > static_cast<size_t>(n_gram_size)) window.erase(window.begin());
|
| }
|
| }
|
|
|
| static vector<StrPtr>
|
| intern_tokens(KnowledgeBase &kb, const vector<string> &tokens)
|
| {
|
| vector<StrPtr> out;
|
| out.reserve(tokens.size());
|
| for (const auto &t : tokens) out.push_back(kb.interner.intern(t));
|
| return out;
|
| }
|
|
|
|
|
| static const NextSet* lookup_ngram_with_backoff(
|
| const KnowledgeBase &kb,
|
| const vector<StrPtr> &ctx_path,
|
| int n_gram_size)
|
| {
|
| int cur_len = min(static_cast<int>(ctx_path.size()), n_gram_size);
|
| while (cur_len > 0) {
|
| vector<StrPtr> window(ctx_path.end() - cur_len, ctx_path.end());
|
| const NextSet* res = kb.lookup_ngram_fast(window);
|
| if (res && !res->empty()) {
|
| return res;
|
| }
|
| cur_len--;
|
| }
|
| return nullptr;
|
| }
|
|
|
| struct CandState {
|
| StrPtr orig_cand;
|
| vector<vector<StrPtr>> frontier;
|
| double total_score;
|
| };
|
|
|
| static string choose_candidate(
|
| const NextSet &cands,
|
| const vector<StrPtr> &history_ptrs,
|
| const KnowledgeBase &kb,
|
| const unordered_map<string, int> &recent_counts,
|
| double repeat_penalty,
|
| int n_gram_size)
|
| {
|
| if (cands.empty()) return string();
|
| if (cands.size() == 1) return *cands[0];
|
|
|
|
|
| const size_t words = max<size_t>(1, (kb.interner.size() + 63u) / 64u);
|
| vector<uint64_t> agg_words(words, 0ULL);
|
|
|
| for (StrPtr t : history_ptrs) {
|
| TokenId tid = kb.interner.id_of(t);
|
| if (tid != TOKEN_ID_INVALID) bitset_set(agg_words.data(), words, tid);
|
| }
|
| const size_t agg_count = bitset_count(agg_words.data(), words);
|
|
|
|
|
| vector<StrPtr> base_ctx;
|
| int base_len = min(static_cast<int>(history_ptrs.size()), n_gram_size - 1);
|
| if (base_len > 0) {
|
| base_ctx.assign(history_ptrs.end() - base_len, history_ptrs.end());
|
| }
|
|
|
|
|
| vector<CandState> tied_states(cands.size());
|
| for (size_t i = 0; i < cands.size(); ++i) {
|
| tied_states[i].orig_cand = cands[i];
|
|
|
|
|
| vector<StrPtr> init_path = base_ctx;
|
| init_path.push_back(cands[i]);
|
| if (init_path.size() > static_cast<size_t>(n_gram_size)) {
|
| init_path.erase(init_path.begin(), init_path.end() - n_gram_size);
|
| }
|
|
|
| tied_states[i].frontier.push_back(move(init_path));
|
| tied_states[i].total_score = 0.0;
|
| }
|
|
|
|
|
| for (int depth = 1; depth <= n_gram_size; ++depth) {
|
|
|
| #ifdef _OPENMP
|
| #pragma omp parallel for schedule(dynamic)
|
| #endif
|
| for (ptrdiff_t i = 0; i < static_cast<ptrdiff_t>(tied_states.size()); ++i) {
|
| double depth_max_score = -1.0;
|
| vector<vector<StrPtr>> next_frontier;
|
|
|
| for (const auto &path : tied_states[i].frontier) {
|
|
|
| const NextSet* opt_next = lookup_ngram_with_backoff(kb, path, n_gram_size);
|
| if (!opt_next || opt_next->empty()) continue;
|
|
|
| size_t count = 0;
|
| size_t inter = 0;
|
|
|
|
|
| for (StrPtr nxt : *opt_next) {
|
| TokenId nid = kb.interner.id_of(nxt);
|
| if (nid != TOKEN_ID_INVALID) {
|
| count++;
|
| if ((agg_words[nid >> 6] & (1ULL << (nid & 63u))) != 0) {
|
| inter++;
|
| }
|
| }
|
| }
|
|
|
| const size_t uni = agg_count + count - inter;
|
| const double score = uni ? static_cast<double>(inter) / static_cast<double>(uni) : 0.0;
|
|
|
| if (score > depth_max_score) {
|
| depth_max_score = score;
|
| }
|
|
|
|
|
| if (depth < n_gram_size) {
|
| for (StrPtr nxt : *opt_next) {
|
| vector<StrPtr> next_path = path;
|
| next_path.push_back(nxt);
|
| if (next_path.size() > static_cast<size_t>(n_gram_size)) {
|
| next_path.erase(next_path.begin(), next_path.end() - n_gram_size);
|
| }
|
| next_frontier.push_back(move(next_path));
|
| }
|
| }
|
| }
|
|
|
|
|
| if (depth == 1) {
|
| const string &tok = *(tied_states[i].orig_cand);
|
| auto rc_it = recent_counts.find(tok);
|
| const int cnt = (rc_it == recent_counts.end()) ? 0 : rc_it->second;
|
| if (depth_max_score > -0.5) {
|
| depth_max_score -= repeat_penalty * static_cast<double>(cnt);
|
| } else {
|
| depth_max_score = -repeat_penalty * static_cast<double>(cnt);
|
| }
|
| } else {
|
| if (depth_max_score < 0.0) {
|
| depth_max_score = 0.0;
|
| }
|
| }
|
|
|
|
|
| tied_states[i].total_score += depth_max_score;
|
|
|
|
|
| if (next_frontier.size() > 1) {
|
| sort(next_frontier.begin(), next_frontier.end());
|
| next_frontier.erase(unique(next_frontier.begin(), next_frontier.end()), next_frontier.end());
|
| }
|
| tied_states[i].frontier = move(next_frontier);
|
| }
|
| }
|
|
|
|
|
| double best_total = -1e9;
|
| for (const auto &st : tied_states) {
|
| if (st.total_score > best_total) {
|
| best_total = st.total_score;
|
| }
|
| }
|
|
|
|
|
| vector<StrPtr> tied_cands;
|
| for (const auto &st : tied_states) {
|
| if (st.total_score >= best_total - 1e-9) {
|
| tied_cands.push_back(st.orig_cand);
|
| }
|
| }
|
|
|
|
|
| string best_tok = *(tied_cands[0]);
|
| for (size_t i = 1; i < tied_cands.size(); ++i) {
|
| if (*(tied_cands[i]) < best_tok) {
|
| best_tok = *(tied_cands[i]);
|
| }
|
| }
|
|
|
| return best_tok;
|
| }
|
|
|
| static vector<string> construct_response(KnowledgeBase &kb,
|
| const vector<string> &prompt_toks,
|
| size_t response_maxlen,
|
| double repeat_penalty,
|
| int n_gram_size,
|
| bool allow_path_loops)
|
| {
|
| vector<string> resp;
|
| if (prompt_toks.empty() || response_maxlen == 0) return resp;
|
|
|
| auto prompt_ptrs = intern_tokens(kb, prompt_toks);
|
| unordered_map<string, int> recent_counts;
|
|
|
|
|
| vector<StrPtr> context = prompt_ptrs;
|
|
|
| auto would_create_2_cycle = [&](const string &cand) -> bool {
|
| if (resp.size() < 3) return false;
|
| return cand == resp[resp.size() - 2] &&
|
| resp.back() == resp[resp.size() - 3];
|
| };
|
|
|
| for (size_t step = 0; step < response_maxlen; ++step){
|
| NextSet candidates;
|
| bool found = false;
|
|
|
| int current_n = min(static_cast<int>(context.size()), n_gram_size);
|
| while (current_n > 0) {
|
| vector<StrPtr> search_ctx(context.end() - current_n, context.end());
|
| auto opt = kb.lookup_ngram(search_ctx);
|
| if (opt && !opt->empty()) {
|
| candidates = *opt;
|
| found = true;
|
| break;
|
| }
|
| current_n--;
|
| }
|
|
|
| if (!found || candidates.empty()) break;
|
|
|
| if (candidates.size() == 1){
|
| string only = *candidates[0];
|
| if (!allow_path_loops && recent_counts[only] > 0) break;
|
| resp.push_back(only);
|
| StrPtr ptr = kb.interner.intern(only);
|
| context.push_back(ptr);
|
| recent_counts[only] += 1;
|
| cout << only << ' ' << flush;
|
| continue;
|
| }
|
|
|
| string chosen = choose_candidate(
|
| candidates, context, kb, recent_counts, repeat_penalty, n_gram_size
|
| );
|
|
|
| if (chosen.empty()) break;
|
| if (!allow_path_loops && would_create_2_cycle(chosen)) break;
|
|
|
| resp.push_back(chosen);
|
| StrPtr chosen_ptr = kb.interner.intern(chosen);
|
| context.push_back(chosen_ptr);
|
| recent_counts[chosen] += 1;
|
| cout << chosen << ' ' << flush;
|
| }
|
|
|
| return resp;
|
| }
|
|
|
| static void learn_from_file(KnowledgeBase &kb, const string &fname, int n_gram_size){
|
| ifstream ifs(fname);
|
| if (!ifs) return;
|
| vector<string> tokens;
|
| string tok;
|
| while (ifs >> tok) tokens.push_back(tok);
|
| learn_tokens_ngram(kb, tokens, n_gram_size);
|
| }
|
|
|
| static void learn_files_parallel(KnowledgeBase &kb, const vector<string> &files, int n_gram_size){
|
| #pragma omp parallel for schedule(dynamic)
|
| for (ptrdiff_t i=0;i<static_cast<ptrdiff_t>(files.size());++i){
|
| learn_from_file(kb, files[(size_t)i], n_gram_size);
|
| }
|
| }
|
|
|
| static constexpr uint64_t KB_MAGIC = 0x434850434B535641ULL;
|
| static constexpr uint64_t KB_VERSION = 2ULL;
|
|
|
| static void write_u64(ostream &os, uint64_t v) {
|
| do {
|
| unsigned char byte = v & 0x7F;
|
| v >>= 7;
|
| if (v != 0) byte |= 0x80;
|
| os.put(static_cast<char>(byte));
|
| } while (v != 0);
|
| if (!os) throw runtime_error("write_u64 failed");
|
| }
|
|
|
| static uint64_t read_u64(istream &is) {
|
| uint64_t v = 0;
|
| unsigned int shift = 0;
|
| unsigned char byte;
|
| do {
|
| if (!is.read(reinterpret_cast<char*>(&byte), 1)) {
|
| throw runtime_error("read_u64 failed");
|
| }
|
| v |= (static_cast<uint64_t>(byte & 0x7F) << shift);
|
| shift += 7;
|
| } while (byte & 0x80);
|
| return v;
|
| }
|
|
|
| static void write_string(ostream &os, const string &s){
|
| write_u64(os, static_cast<uint64_t>(s.size()));
|
| if (!s.empty()){
|
| os.write(s.data(), static_cast<streamsize>(s.size()));
|
| if(!os) throw runtime_error("write_string failed");
|
| }
|
| }
|
|
|
| static string read_string(istream &is){
|
| uint64_t n = read_u64(is);
|
|
|
|
|
| string s;
|
| s.resize(static_cast<size_t>(n));
|
|
|
| if (n != 0){
|
| is.read(&s[0], static_cast<streamsize>(n));
|
| if(!is) throw runtime_error("read_string failed");
|
| }
|
| return s;
|
| }
|
|
|
| static void save_kb_binary(const KnowledgeBase &kb, const string &fname){
|
| const string temp = fname + ".tmp";
|
|
|
| {
|
| ofstream ofs(temp.c_str(), ios::binary | ios::trunc);
|
| if (!ofs) throw runtime_error("cannot open temp save file");
|
|
|
| vector<string> pool;
|
| pool.reserve(kb.interner.pool.size());
|
| for (const auto &s : kb.interner.pool) pool.push_back(s);
|
|
|
| sort(pool.begin(), pool.end());
|
|
|
| unordered_map<string, uint64_t> id;
|
| id.reserve(pool.size());
|
| for (uint64_t i = 0; i < static_cast<uint64_t>(pool.size()); ++i){
|
| id.emplace(pool[(size_t)i], i);
|
| }
|
|
|
| write_u64(ofs, KB_MAGIC);
|
| write_u64(ofs, KB_VERSION);
|
|
|
| write_u64(ofs, static_cast<uint64_t>(pool.size()));
|
| for (const auto &s : pool) write_string(ofs, s);
|
|
|
| write_u64(ofs, static_cast<uint64_t>(kb.next.size()));
|
| for (const auto &pr : kb.next){
|
| write_u64(ofs, static_cast<uint64_t>(pr.first.size()));
|
| for (StrPtr ctx_tok : pr.first){
|
| write_u64(ofs, id.at(*ctx_tok));
|
| }
|
| write_u64(ofs, static_cast<uint64_t>(pr.second.size()));
|
| for (StrPtr nxt : pr.second){
|
| write_u64(ofs, id.at(*nxt));
|
| }
|
| }
|
|
|
| ofs.flush();
|
| if (!ofs) throw runtime_error("failed while writing temp save file");
|
| }
|
|
|
| remove(fname.c_str());
|
| if (rename(temp.c_str(), fname.c_str()) != 0){
|
| remove(temp.c_str());
|
| throw runtime_error("failed to commit save file");
|
| }
|
| }
|
|
|
| static void load_kb_binary(KnowledgeBase &kb, const string &fname){
|
| ifstream ifs(fname, ios::binary);
|
| if (!ifs) throw runtime_error("cannot open load file");
|
|
|
| const uint64_t magic = read_u64(ifs);
|
| if (magic != KB_MAGIC) throw runtime_error("bad save file magic");
|
|
|
| const uint64_t version = read_u64(ifs);
|
| if (version != KB_VERSION) throw runtime_error("unsupported save file version");
|
|
|
| const uint64_t N = read_u64(ifs);
|
|
|
|
|
| vector<string> strings;
|
| strings.reserve(static_cast<size_t>(N));
|
|
|
| for (uint64_t i = 0; i < N; ++i){
|
| strings.push_back(read_string(ifs));
|
| }
|
|
|
| kb.interner.pool.clear();
|
| kb.interner.pool.reserve(static_cast<size_t>(N));
|
|
|
| vector<StrPtr> ptrs;
|
| ptrs.reserve(static_cast<size_t>(N));
|
| for (const auto &s : strings){
|
| ptrs.push_back(kb.interner.intern(s));
|
| }
|
|
|
|
|
| const uint64_t E = read_u64(ifs);
|
|
|
|
|
| {
|
| lock_guard<mutex> lk(kb.m);
|
| kb.next.clear();
|
| kb.next.reserve(static_cast<size_t>(E));
|
| }
|
|
|
| for (uint64_t i = 0; i < E; ++i){
|
| const uint64_t ctx_size = read_u64(ifs);
|
|
|
|
|
| vector<StrPtr> ctx;
|
| ctx.reserve(static_cast<size_t>(ctx_size));
|
| for(uint64_t c = 0; c < ctx_size; ++c) {
|
| const uint64_t c_idx = read_u64(ifs);
|
| if (c_idx >= ptrs.size()) throw runtime_error("save file is corrupted: bad context key");
|
| ctx.push_back(ptrs[(size_t)c_idx]);
|
| }
|
|
|
| const uint64_t M = read_u64(ifs);
|
|
|
|
|
| NextSet vec;
|
| vec.reserve(static_cast<size_t>(M));
|
|
|
| for (uint64_t j = 0; j < M; ++j){
|
| const uint64_t v_idx = read_u64(ifs);
|
| if (v_idx >= ptrs.size()) throw runtime_error("save file is corrupted: bad graph value");
|
| vec.push_back(ptrs[(size_t)v_idx]);
|
| }
|
|
|
| {
|
| lock_guard<mutex> lk(kb.m);
|
| kb.next.emplace(move(ctx), move(vec));
|
| }
|
| }
|
| }
|
|
|
| static void print_kb_info(const string &fname) {
|
| ifstream ifs(fname, ios::binary);
|
| if (!ifs) {
|
| cerr << "Error: cannot open knowledge-base file " << fname << "\n";
|
| return;
|
| }
|
|
|
| cout << "Reckoning knowledge-base " << fname << " info. ";
|
|
|
| try {
|
| const uint64_t magic = read_u64(ifs);
|
| if (magic != KB_MAGIC) throw runtime_error("bad save file magic");
|
|
|
| const uint64_t version = read_u64(ifs);
|
| if (version != KB_VERSION) throw runtime_error("unsupported save file version");
|
|
|
| const uint64_t N = read_u64(ifs);
|
|
|
|
|
|
|
| for (uint64_t i = 0; i < N; ++i){
|
| uint64_t len = read_u64(ifs);
|
|
|
| if (len > 0) ifs.seekg(static_cast<streamoff>(len), ios_base::cur);
|
| }
|
|
|
| const uint64_t E = read_u64(ifs);
|
|
|
|
|
| uint64_t max_ngram = 0;
|
| for (uint64_t i = 0; i < E; ++i){
|
| const uint64_t ctx_size = read_u64(ifs);
|
|
|
|
|
| if (ctx_size > max_ngram) max_ngram = ctx_size;
|
|
|
|
|
| for(uint64_t c = 0; c < ctx_size; ++c) {
|
| read_u64(ifs);
|
| }
|
|
|
| const uint64_t M = read_u64(ifs);
|
|
|
|
|
|
|
| for(uint64_t j = 0; j < M; ++j){
|
| read_u64(ifs);
|
| }
|
| }
|
|
|
| cout << "The " << fname << " n-gram max size is " << max_ngram << "\n";
|
|
|
| } catch (const exception &e) {
|
| cerr << "Error reading knowledge-base info: " << e.what() << "\n";
|
| }
|
| }
|
|
|
| static void print_commands(const char *p){
|
| cout << p << " command-line interface options:\n";
|
| cout << " --response-max-length N Set maximum number of token(s) in a response. Default number is 500\n";
|
| cout << " --save-kb FILE Save the knowledge-base graph to a binary file.\n";
|
| cout << " --load-kb FILE Load a previously saved knowledge-base graph from a binary file.\n";
|
| cout << " --info-kb FILE Show the dictionary-depth and n-gram max size of a knowledge-base file.\n";
|
| cout << " --static-kb Disable learning prompt(s) and response(s) during interactive session.\n";
|
| cout << " --n-gram-max-size N Set maximum size of the n-gram where N is the size. Default size is 3\n";
|
| cout << " --learn f1 f2 ... Learn from text file(s) to update the knowledge-base graph.\n";
|
| cout << " --repeat-penalty P Set penalty for repeated tokens in response. Default penalty is 0.7 (higher penalty reduce repetition).\n";
|
| cout << " --allow-path-loops Allow finite loops when the knowledge-base graph contains a circular single-path reference.\n";
|
| cout << " --activate-agi Activate the Artificial General Intelligence (AGI) features.\n";
|
| cout << " --help Show " << p << " command-line interface options.\n";
|
| }
|
|
|
| int main(int argc, char **argv){
|
| size_t response_maxlen = 500;
|
| string savefile;
|
| string load_kb;
|
| int n_gram_size = 3;
|
| double repeat_penalty = 0.7;
|
| vector<string> learn_files;
|
| bool learn_chat = true;
|
| bool allow_path_loops = false;
|
|
|
| for (int i=1;i<argc;++i){
|
| string a = argv[i];
|
| if (a=="--help"){ print_commands(argv[0]); return 0; }
|
| if (a=="--response-max-length" && i+1<argc){ response_maxlen = stoul(argv[++i]); continue; }
|
| if (a=="--save-kb" && i+1<argc){ savefile = argv[++i]; continue; }
|
| if (a=="--load-kb" && i+1<argc){ load_kb = argv[++i]; continue; }
|
| if (a=="--info-kb" && i+1<argc){ print_kb_info(argv[++i]); return 0; }
|
| if (a=="--static-kb"){ learn_chat = false; continue; }
|
| if (a=="--n-gram-max-size" && i+1<argc){ n_gram_size = max(1, stoi(argv[++i])); continue; }
|
| if (a=="--learn"){ while(i+1<argc && argv[i+1][0] != '-') learn_files.push_back(argv[++i]); continue; }
|
| if (a=="--repeat-penalty" && i+1<argc){ repeat_penalty = stod(argv[++i]); continue; }
|
| if (a=="--allow-path-loops") { allow_path_loops = true; continue; }
|
| if (a=="--activate-agi"){
|
| cerr << "AGI subscription fee is $1,000 CAD per month. To pay the fee, contact Caleb Nwokocha via email cnwokocha@proton.me\n";
|
| return 0;
|
| }
|
|
|
| cerr << "Error: Unknown or incomplete command-line option '" << a << "'\n";
|
| print_commands(argv[0]);
|
| return 1;
|
| }
|
|
|
| KnowledgeBase kb;
|
|
|
| if (!load_kb.empty()){
|
| try { print_kb_info(load_kb);
|
| cerr << "Loading knowledge base " << load_kb << " using values from " << argv[0] << " command-line interface options: --n-gram-max-size " << n_gram_size <<
|
| ", --response-max-length " << response_maxlen << ", and --repeat-penalty " << repeat_penalty << "\n";
|
| cerr << "To change the value(s), use " << argv[0] << " command-line interface option(s).\n";
|
| load_kb_binary(kb, load_kb); cerr << "Loaded knowledge base: " << load_kb << "\n"; }
|
| catch (const exception &e){ cerr << "Error: " << e.what() << "\n"; }
|
| }
|
|
|
| if (!learn_files.empty()){
|
| cerr << "Learning from file/s (" << learn_files.size() << ") using threads=" << omp_get_max_threads() << "\n";
|
| learn_files_parallel(kb, learn_files, n_gram_size);
|
| }
|
|
|
| string line;
|
| cout << "Ready. Enter prompts.\n";
|
| while (cout << "> " , getline(cin, line)){
|
| if (line.empty()){ cout << "\n"; continue; }
|
| auto prompt_toks = tokenize_whitespace(line);
|
| if (learn_chat) {learn_tokens_ngram(kb, prompt_toks, n_gram_size);}
|
| auto resp = construct_response(kb, prompt_toks, response_maxlen, repeat_penalty, n_gram_size, allow_path_loops);
|
| cout << "\n";
|
| if (!resp.empty() && learn_chat){learn_tokens_ngram(kb, resp, n_gram_size);}
|
| if (!savefile.empty()){
|
| try { cerr << "Saving knowledge base: " << savefile << "\n";
|
| save_kb_binary(kb, savefile); cerr << "Saved knowledge base: " << savefile << "\n"; }
|
| catch (const exception &e){ cerr << "Error: " << e.what() << "\n"; }
|
| }
|
| }
|
|
|
| return 0;
|
| }
|
|
|
