File size: 2,575 Bytes
92264aa | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 | /**************************************************************************************************
* ZipVoice AXERA C++ Port
*
* Tokenizer: loads tokens.txt and maps token strings to IDs.
*
* For board deployment, tokenization should ideally be done on a host machine
* and token IDs passed directly. This class provides basic tokenization
* capability for convenience.
**************************************************************************************************/
#pragma once
#include <string>
#include <vector>
#include <unordered_map>
#include <cstdint>
class Tokenizer {
public:
Tokenizer();
/**
* Load tokens.txt file. Format: token\tid per line.
*/
int Load(const std::string& token_file);
/**
* Convert a single text to token IDs using simplified rules.
* For production use, pre-compute token IDs with the Python tokenizer.
*/
std::vector<int> TextToTokenIds(const std::string& text) const;
/**
* Convert multiple texts to token IDs.
*/
std::vector<std::vector<int>> TextsToTokenIds(const std::vector<std::string>& texts) const;
/**
* Get the pad token ID ("_" token).
*/
int GetPadId() const { return m_pad_id; }
/**
* Get vocabulary size.
*/
int GetVocabSize() const { return static_cast<int>(m_token2id.size()); }
/**
* Lookup a single token. Returns pad_id if not found.
*/
int TokenToId(const std::string& token) const;
/**
* Build concatenated token array [prompt_tokens + text_tokens + pad] padded to max_tokens.
*/
void BuildCatTokens(const std::vector<int>& prompt_tokens,
const std::vector<int>& text_tokens,
int max_tokens,
std::vector<int32_t>& out_cat_tokens) const;
bool IsLoaded() const { return m_loaded; }
private:
std::unordered_map<std::string, int> m_token2id;
std::vector<std::string> m_id2token;
int m_pad_id;
bool m_loaded;
// Simplified text→tokens for Chinese (char-by-char pinyin-like tokens)
std::vector<std::string> TokenizeZh(const std::string& text) const;
// Simplified text→tokens for English (basic phoneme rules)
std::vector<std::string> TokenizeEn(const std::string& text) const;
// Split text into language segments
std::vector<std::pair<std::string, std::string>> GetSegments(const std::string& text) const;
static bool IsChinese(char c);
static bool IsAlphabet(char c);
static std::string MapPunctuation(const std::string& text);
};
|