| // | |
| // common_ngram_mod | |
| // | |
| common_ngram_mod::common_ngram_mod(uint16_t n, size_t size) : n(n), used(0) { | |
| entries.resize(size); | |
| reset(); | |
| } | |
| size_t common_ngram_mod::idx(const entry_t * tokens) const { | |
| size_t res = 0; | |
| for (size_t i = 0; i < n; ++i) { | |
| res = res*6364136223846793005ULL + tokens[i]; | |
| } | |
| res = res % entries.size(); | |
| return res; | |
| } | |
| void common_ngram_mod::add(const entry_t * tokens) { | |
| const size_t i = idx(tokens); | |
| if (entries[i] == EMPTY) { | |
| used++; | |
| } | |
| entries[i] = tokens[n]; | |
| } | |
| common_ngram_mod::entry_t common_ngram_mod::get(const entry_t * tokens) const { | |
| const size_t i = idx(tokens); | |
| return entries[i]; | |
| } | |
| void common_ngram_mod::reset() { | |
| std::fill(entries.begin(), entries.end(), EMPTY); | |
| used = 0; | |
| } | |
| size_t common_ngram_mod::get_n() const { | |
| return n; | |
| } | |
| size_t common_ngram_mod::get_used() const { | |
| return used; | |
| } | |
| size_t common_ngram_mod::size() const { | |
| return entries.size(); | |
| } | |
| size_t common_ngram_mod::size_bytes() const { | |
| return entries.size() * sizeof(entries[0]); | |
| } | |