alter_ego / serving /src /tokenizer_bpe.rs
geraldaton20's picture
initial upload: source code (no build artifacts)
f4f166e verified
Raw History Blame Contribute Delete
2.03 kB
//! HF `tokenizer.json` parity layer: loads the exact BPE trained by Stage-3
//! (`preprocess/tokenizer.json`) so Rust encode/decode matches the training
//! corpus byte-for-byte. Implements digestion's Tokenizer (encode) and the
//! Detokenizer needed to turn student generations back into text.
use crate::digestion::Tokenizer;
use crate::error::{Error, Result};
pub trait Detokenizer: Send + Sync {
fn decode(&self, ids: &[u32]) -> String;
fn vocab_size(&self) -> usize;
}
pub struct HfBpe {
inner: tokenizers::Tokenizer,
}
impl HfBpe {
pub fn from_file(path: &str) -> Result<Self> {
let inner = tokenizers::Tokenizer::from_file(path)
.map_err(|e| Error::Model(format!("tokenizer load failed: {e}")))?;
Ok(Self { inner })
}
/// Raw access for advanced paths (offset mappings, ONNX pipelines).
pub fn raw(&self) -> &tokenizers::Tokenizer {
&self.inner
}
}
impl Tokenizer for HfBpe {
fn encode(&self, text: &str) -> Vec<u32> {
self.inner
.encode(text, false)
.map(|e| e.get_ids().to_vec())
.unwrap_or_default()
}
}
impl Detokenizer for HfBpe {
fn decode(&self, ids: &[u32]) -> String {
let u: Vec<u32> = ids.to_vec();
self.inner.decode(&u, false).unwrap_or_default()
}
fn vocab_size(&self) -> usize {
self.inner.get_vocab_size(true)
}
}
/// Dev fallback: cannot invert hashed ids; returns bracketed ids so callers
/// fail loudly instead of silently producing garbage text.
pub struct StubDetok;
impl Detokenizer for StubDetok {
fn decode(&self, ids: &[u32]) -> String {
format!("[tok:{}]", ids.iter().map(|i| i.to_string()).collect::<Vec<_>>().join(","))
}
fn vocab_size(&self) -> usize {
0
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn stub_detok_roundtrip_shape() {
let d = StubDetok;
let s = d.decode(&[1, 2, 3]);
assert!(s.starts_with("[tok:") && s.ends_with(']'));
}
}