include/ling3/tokenizer.h

#pragma once

#include <cstddef>
#include <cstdint>
#include <memory>
#include <span>
#include <string>
#include <string_view>
#include <vector>

namespace ling3 {

class Tokenizer {
public:
    explicit Tokenizer(std::span<const std::byte> data);
    ~Tokenizer();

    Tokenizer(const Tokenizer &) = delete;
    Tokenizer & operator=(const Tokenizer &) = delete;
    Tokenizer(Tokenizer &&) noexcept;
    Tokenizer & operator=(Tokenizer &&) noexcept;

    std::vector<std::uint32_t> Encode(std::string_view text) const;
    std::string Decode(std::span<const std::uint32_t> tokens, bool skip_special = false) const;
    std::string_view Piece(std::uint32_t token) const;
    std::size_t vocab_size() const noexcept;
    bool uses_official_unicode_rules() const noexcept;

private:
    struct Impl;
    std::unique_ptr<Impl> impl_;
};

} // namespace ling3