//! HF `tokenizer.json` parity layer: loads the exact BPE trained by Stage-3 //! (`preprocess/tokenizer.json`) so Rust encode/decode matches the training //! corpus byte-for-byte. Implements digestion's Tokenizer (encode) and the //! Detokenizer needed to turn student generations back into text. use crate::digestion::Tokenizer; use crate::error::{Error, Result}; pub trait Detokenizer: Send + Sync { fn decode(&self, ids: &[u32]) -> String; fn vocab_size(&self) -> usize; } pub struct HfBpe { inner: tokenizers::Tokenizer, } impl HfBpe { pub fn from_file(path: &str) -> Result { let inner = tokenizers::Tokenizer::from_file(path) .map_err(|e| Error::Model(format!("tokenizer load failed: {e}")))?; Ok(Self { inner }) } /// Raw access for advanced paths (offset mappings, ONNX pipelines). pub fn raw(&self) -> &tokenizers::Tokenizer { &self.inner } } impl Tokenizer for HfBpe { fn encode(&self, text: &str) -> Vec { self.inner .encode(text, false) .map(|e| e.get_ids().to_vec()) .unwrap_or_default() } } impl Detokenizer for HfBpe { fn decode(&self, ids: &[u32]) -> String { let u: Vec = ids.to_vec(); self.inner.decode(&u, false).unwrap_or_default() } fn vocab_size(&self) -> usize { self.inner.get_vocab_size(true) } } /// Dev fallback: cannot invert hashed ids; returns bracketed ids so callers /// fail loudly instead of silently producing garbage text. pub struct StubDetok; impl Detokenizer for StubDetok { fn decode(&self, ids: &[u32]) -> String { format!("[tok:{}]", ids.iter().map(|i| i.to_string()).collect::>().join(",")) } fn vocab_size(&self) -> usize { 0 } } #[cfg(test)] mod tests { use super::*; #[test] fn stub_detok_roundtrip_shape() { let d = StubDetok; let s = d.decode(&[1, 2, 3]); assert!(s.starts_with("[tok:") && s.ends_with(']')); } }