File size: 858 Bytes
3fd1a35 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 | #pragma once
#include <cstddef>
#include <cstdint>
#include <memory>
#include <span>
#include <string>
#include <string_view>
#include <vector>
namespace ling3 {
class Tokenizer {
public:
explicit Tokenizer(std::span<const std::byte> data);
~Tokenizer();
Tokenizer(const Tokenizer &) = delete;
Tokenizer & operator=(const Tokenizer &) = delete;
Tokenizer(Tokenizer &&) noexcept;
Tokenizer & operator=(Tokenizer &&) noexcept;
std::vector<std::uint32_t> Encode(std::string_view text) const;
std::string Decode(std::span<const std::uint32_t> tokens, bool skip_special = false) const;
std::string_view Piece(std::uint32_t token) const;
std::size_t vocab_size() const noexcept;
bool uses_official_unicode_rules() const noexcept;
private:
struct Impl;
std::unique_ptr<Impl> impl_;
};
} // namespace ling3
|