//! Self-contained, byte-exact Turkish morphological tokenizer. #![forbid(unsafe_code)] use core::fmt; use std::{ collections::HashMap, ops::Range, sync::{ atomic::{AtomicUsize, Ordering}, Arc, }, thread, }; use nedo_core::{scan, scan_compact, CompactScanResult, LexicalKind, ScanError}; use nedo_format::{ByteSpan, FormatError, LosslessDocument, SurfaceUnit}; use nedo_morph_bundle::{ ambiguity_word_data, AmbiguityScoringCode, AmbiguityWordData, BinaryError, DisambiguationError, DisambiguationScoreCache, NativeAnalysis, NativeDisambiguator, NativeMorpheme, NativeMorphology, SharedPreparedWordCache, }; use sha2::{Digest, Sha256}; mod alignment; mod code; mod codec; #[cfg(feature = "compiled-surface-table")] mod compiled_analysis_table; mod flat_surface; mod nedoformer; mod nedoformer_contract; mod nedoformer_input; pub(crate) mod nedoformer_sidecar; mod nedoformer_vocab; mod quality; mod sharded_surface; mod surface_vocab; mod training; mod vocab; pub use codec::{decode as decode_tokenized, encode as encode_tokenized}; #[cfg(feature = "compiled-surface-table")] pub use compiled_analysis_table::{ encode_compiled_surface_analysis_table, CompiledSurfaceAnalysisEntry, CompiledSurfaceAnalysisTable, CompiledSurfaceAnalysisTableError, CompiledSurfaceCandidateEntry, }; pub use nedo_format::{decode_binary as decode_lossless, encode_binary as encode_lossless}; pub use nedoformer::{ NedoFormerLatticeDocument, NedoFormerLatticeUnit, NedoFormerSamplingPolicy, NedoFormerSegmentationCandidate, }; pub use nedoformer_contract::{ NedoFormerContractFingerprint, NEDOFORMER_CODE_SEGMENTATION_VERSION, NEDOFORMER_INPUT_ENCODING_VERSION, NEDOFORMER_LATTICE_SCHEMA_VERSION, NEDOFORMER_NUMERIC_SEGMENTATION_VERSION, NEDOFORMER_SHADOW_NORMALIZATION_VERSION, NEDOFORMER_TOKENIZER_CONTRACT_VERSION, NEDOFORMER_WHITESPACE_SCHEMA_VERSION, }; pub use nedoformer_input::NedoFormerInputEncoding; pub use nedoformer_sidecar::{ NedoFormerLatticeSidecar, NedoFormerSidecarCandidate, NedoFormerSidecarUnit, NEDOFORMER_SIDECAR_SCHEMA_VERSION, }; pub use nedoformer_vocab::{ NedoFormerGenerationEncoding, NedoFormerVocabEntry, NedoFormerVocabKind, NedoFormerVocabulary, }; pub use sharded_surface::{ merge_surface_shards, ShardedSurfaceEncoder, ShardedSurfaceRuntimeCache, SurfaceShardBatch, }; pub use surface_vocab::{ SurfaceVocabulary, SurfaceVocabularyTrainer, SURFACE_BOS_ID, SURFACE_BYTE_BASE_ID, SURFACE_ENTRY_BASE_ID, SURFACE_EOS_ID, SURFACE_PAD_ID, }; pub use training::{ encode_byte_training_document, TrainingBatch, TrainingEncoding, TrainingEncodingOptions, }; pub use vocab::{ CharacterVocabulary, GenerationVocabulary, BOS_ID, BYTE_BASE_ID, CHAR_BASE_ID, CODE_END_ID, CODE_START_ID, EOS_ID, MORPHEME_BOUNDARY_ID, PAD_ID, UNIT_BOUNDARY_ID, }; use alignment::align_analysis; use code::{auto_code_spans, explicit_code_spans, identifier_cuts}; use quality::{analyze_token_with_nedoformer_shadow, analyze_token_with_quality_fallback}; /// Stable tokenized-document schema version. pub const TOKENIZER_SCHEMA_VERSION: u32 = 1; /// Embedded native morphology SHA-256. pub const MORPHOLOGY_SHA256: &str = "8b78ba8c6352e2abba8b7148da143896257606e916693f640649b2b57a0a23d6"; /// Embedded native disambiguation model SHA-256. pub const MODEL_SHA256: &str = "1aaaaf5343fda967d5864dcae4b44540c25ba859c385c5f103aaabc988056ee0"; const EMBEDDED_MORPHOLOGY: &[u8] = include_bytes!("../../../assets/native/nedo-morph-v1.bin"); const EMBEDDED_MODEL: &[u8] = include_bytes!("../../../assets/native/model-compressed"); const MAX_SENTENCE_TOKENS: usize = 512; const BATCH_ANALYSIS_CACHE_ENTRIES: usize = 65_536; const LOW_PARALLEL_ANALYSIS_CACHE_BUDGET: usize = 262_144; fn analysis_cache_entries_for_parallelism(parallelism: usize) -> usize { LOW_PARALLEL_ANALYSIS_CACHE_BUDGET .checked_div(parallelism.max(1)) .unwrap_or(BATCH_ANALYSIS_CACHE_ENTRIES) .max(BATCH_ANALYSIS_CACHE_ENTRIES) } const MIN_BATCH_CHUNK_BYTES: usize = 1 << 20; const BATCH_CHUNKS_PER_WORKER: usize = 16; const SEGMENT_PROGRAM_CACHE_ENTRIES: usize = 32_768; const SURFACE_PROGRAM_TABLE_SLOTS: usize = SEGMENT_PROGRAM_CACHE_ENTRIES * 2; const DOCUMENT_PROGRAM_CACHE_ENTRIES: usize = 8_192; const MAX_FALLBACK_CHARS_CONFIG: usize = 4_096; /// Requested tokenizer mode. #[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] #[repr(u8)] pub enum TokenizerMode { /// Detect explicit and high-confidence unmarked code spans. #[default] Auto = 0, /// Treat the complete input as Turkish/mixed natural language. Turkish = 1, /// Treat the complete input as code and bypass morphology. Code = 2, } /// Effective mode for one exact surface unit. #[derive(Clone, Copy, Debug, Eq, PartialEq)] #[repr(u8)] pub enum TokenMode { /// Native Turkish morphology path. Turkish = 1, /// Exact code/raw character path. Code = 2, /// Invalid UTF-8 or control payload path. Opaque = 3, } impl TryFrom for TokenMode { type Error = TokenizerError; fn try_from(value: u8) -> Result { match value { 1 => Ok(Self::Turkish), 2 => Ok(Self::Code), 3 => Ok(Self::Opaque), _ => Err(TokenizerError::InvalidCodecEnum("token mode", value)), } } } /// Analysis state for one unit; no fallback is disguised as morphology. #[derive(Clone, Copy, Debug, Eq, PartialEq)] #[repr(u8)] pub enum TokenStatus { /// Whitespace, line break, punctuation, symbol, or other structural content. Structural = 1, /// Native morphology and contextual selection succeeded. Morphological = 2, /// Explicit unknown analysis generated by the contextual model. Unknown = 3, /// Code-mode content bypassed morphology by configuration/detection. Code = 4, /// Invalid UTF-8/control content preserved exactly. Opaque = 5, } impl TryFrom for TokenStatus { type Error = TokenizerError; fn try_from(value: u8) -> Result { match value { 1 => Ok(Self::Structural), 2 => Ok(Self::Morphological), 3 => Ok(Self::Unknown), 4 => Ok(Self::Code), 5 => Ok(Self::Opaque), _ => Err(TokenizerError::InvalidCodecEnum("token status", value)), } } } /// Immutable tokenizer behavior configuration. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub struct TokenizerConfig { /// Input/code mode policy. pub mode: TokenizerMode, /// Maximum contextual sentence token count before a deterministic chunk boundary. pub max_sentence_tokens: usize, /// Maximum Unicode-scalar count for unknown/code fallback pieces. pub max_fallback_chars: usize, /// Whether sentence context selects among morphological candidates. pub contextual_disambiguation: bool, /// Whether `Auto` mode detects high-confidence unmarked code lines/blocks. pub detect_unmarked_code: bool, } impl Default for TokenizerConfig { fn default() -> Self { Self { mode: TokenizerMode::Auto, max_sentence_tokens: MAX_SENTENCE_TOKENS, max_fallback_chars: 48, contextual_disambiguation: true, detect_unmarked_code: true, } } } /// Exact contextual-score cache policy for the final surface encoder. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum ContextCachePolicy { /// Largest cache: best retention, highest worker-local memory use. Full, /// Intermediate cache sized for high-shard exact preprocessing. Balanced, /// Smallest cache: minimum memory, more score recomputation after saturation. Compact, } /// Runtime policy for the final surface-piece encoder. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub struct SurfaceEncoderOptions { /// Whether Turkish morphology contributes internal surface boundaries. pub use_morphology: bool, /// Maximum exact whole-document programs retained per worker or shard. /// /// Set this to zero for a one-pass corpus build. Word, morphology, segment, /// and surface-program caches remain enabled; only exact document replay is /// disabled, avoiding a persistent copy of every new document and output. pub document_cache_entries: usize, /// Whether contextual trigram scores use the smaller exact shard-local table. /// /// When full, new scores are computed without retention; output is unchanged. pub context_cache_policy: ContextCachePolicy, } impl SurfaceEncoderOptions { /// Standard reusable-service policy with exact document replay enabled. #[must_use] pub const fn cached(use_morphology: bool) -> Self { Self { use_morphology, document_cache_entries: DOCUMENT_PROGRAM_CACHE_ENTRIES, context_cache_policy: ContextCachePolicy::Full, } } /// High-throughput one-pass corpus policy that does not retain complete documents. #[must_use] pub const fn one_pass(use_morphology: bool) -> Self { Self { use_morphology, document_cache_entries: 0, context_cache_policy: ContextCachePolicy::Full, } } /// Balanced one-pass policy for high-shard preprocessing. #[must_use] pub const fn one_pass_balanced(use_morphology: bool) -> Self { Self { use_morphology, document_cache_entries: 0, context_cache_policy: ContextCachePolicy::Balanced, } } /// Lower-memory one-pass policy with a compact exact contextual-score cache. /// /// This can improve eight-shard throughput and memory use, but may reduce /// throughput at higher shard counts. Capacity exhaustion never changes output. #[must_use] pub const fn one_pass_compact(use_morphology: bool) -> Self { Self { use_morphology, document_cache_entries: 0, context_cache_policy: ContextCachePolicy::Compact, } } } impl Default for SurfaceEncoderOptions { fn default() -> Self { Self::cached(true) } } /// One aligned morpheme over original bytes. #[derive(Clone, Debug, Eq, PartialEq)] pub struct AlignedMorpheme { /// Stable morpheme ID. pub id: String, /// Normalized realized surface. pub surface: String, /// Original-byte interval consumed by the surface; epsilon spans are empty. pub span: ByteSpan, /// Whether this morpheme starts a derivational group. pub derivational: bool, } /// Selected analysis metadata. #[derive(Clone, Debug, Eq, PartialEq)] pub struct AnalysisMetadata { /// Canonical reference key. pub canonical: String, /// Stable dictionary/root ID. pub dictionary_id: String, /// Perceptron lemma. pub lemma: String, /// Primary POS short name. pub primary_pos: String, /// Secondary POS short name. pub secondary_pos: String, /// Ordered aligned morphemes. pub morphemes: Vec, } /// One exact surface unit. #[derive(Clone, Debug, Eq, PartialEq)] pub struct TokenizedUnit { /// Full original-byte span. pub span: ByteSpan, /// Scanner class. pub kind: LexicalKind, /// Effective processing mode. pub mode: TokenMode, /// Explicit analysis state. pub status: TokenStatus, /// Inner-model group ID; clitic groups may span whitespace units. pub group_id: Option, /// Strictly increasing byte cuts inside `span`. pub cuts: Vec, /// Selected analysis when applicable. pub analysis: Option, } /// Complete byte-exact tokenized document. #[derive(Clone, Debug, Eq, PartialEq)] pub struct TokenizedDocument { raw: Vec, units: Vec, } impl TokenizedDocument { /// Creates and validates a tokenized document. /// /// # Errors /// /// Returns an error if coverage, cuts, status, or analysis spans are invalid. pub fn new(raw: Vec, units: Vec) -> Result { let document = Self { raw, units }; document.validate()?; Ok(document) } /// Exact original bytes. #[must_use] pub fn raw(&self) -> &[u8] { &self.raw } /// Exact metadata units. #[must_use] pub fn units(&self) -> &[TokenizedUnit] { &self.units } /// Byte-exact decode. #[must_use] pub fn decode(&self) -> &[u8] { &self.raw } /// Converts to the lower-level lossless span/cut document. /// /// # Errors /// /// Returns an error if stored unit spans or cuts violate the lower-level format. pub fn lossless_document(&self) -> Result { let units = self .units .iter() .map(|unit| SurfaceUnit::new(unit.span, unit.cuts.clone())) .collect::, _>>()?; Ok(LosslessDocument::new(self.raw.clone(), units)?) } /// Validates exact coverage, cuts, analysis spans, and status invariants. /// /// # Errors /// /// Returns an error for any coverage, metadata, status, or span inconsistency. pub fn validate(&self) -> Result<(), TokenizerError> { let mut expected = 0_u64; let document_len = u64::try_from(self.raw.len()) .map_err(|_| TokenizerError::LengthOverflow("document length"))?; for (index, unit) in self.units.iter().enumerate() { if unit.span.start != expected || unit.span.end <= unit.span.start { return Err(TokenizerError::InvalidUnitCoverage { index, expected, start: unit.span.start, end: unit.span.end, }); } if unit.span.end > document_len { return Err(TokenizerError::InvalidUnitCoverage { index, expected, start: unit.span.start, end: unit.span.end, }); } let surface = SurfaceUnit::new(unit.span, unit.cuts.clone())?; surface.validate()?; validate_unit_metadata(index, unit)?; expected = unit.span.end; } if expected != document_len { return Err(TokenizerError::IncompleteDocument { covered: expected, document_len, }); } Ok(()) } } fn validate_unit_metadata(index: usize, unit: &TokenizedUnit) -> Result<(), TokenizerError> { let mode_status_valid = matches!( (unit.mode, unit.status), ( TokenMode::Turkish, TokenStatus::Structural | TokenStatus::Morphological | TokenStatus::Unknown ) | (TokenMode::Code, TokenStatus::Code) | (TokenMode::Opaque, TokenStatus::Opaque) ); if !mode_status_valid { return Err(TokenizerError::InvalidUnitMetadata { index, reason: "mode and status are inconsistent", }); } if matches!(unit.kind, LexicalKind::Opaque | LexicalKind::Control) != (unit.mode == TokenMode::Opaque) { return Err(TokenizerError::InvalidUnitMetadata { index, reason: "opaque/control kind and mode are inconsistent", }); } match unit.status { TokenStatus::Morphological | TokenStatus::Unknown => { if unit.analysis.is_none() { return Err(TokenizerError::MissingAnalysis { index }); } } TokenStatus::Structural | TokenStatus::Code | TokenStatus::Opaque => { if unit.analysis.is_some() { return Err(TokenizerError::UnexpectedAnalysis { index }); } } } if let Some(analysis) = &unit.analysis { validate_analysis_metadata(index, unit, analysis)?; } Ok(()) } fn validate_analysis_metadata( index: usize, unit: &TokenizedUnit, analysis: &AnalysisMetadata, ) -> Result<(), TokenizerError> { if analysis.canonical.is_empty() || analysis.dictionary_id.is_empty() || analysis.primary_pos.is_empty() || analysis.morphemes.is_empty() { return Err(TokenizerError::InvalidUnitMetadata { index, reason: "analysis identity is incomplete", }); } let is_unknown = analysis.dictionary_id == "UNK_Unk_Unk"; if (unit.status == TokenStatus::Unknown) != is_unknown { return Err(TokenizerError::InvalidUnitMetadata { index, reason: "unknown status and dictionary identity differ", }); } let mut previous_end = unit.span.start; for morpheme in &analysis.morphemes { if morpheme.span.start < unit.span.start || morpheme.span.end > unit.span.end || morpheme.span.start > morpheme.span.end { return Err(TokenizerError::MorphemeOutsideUnit { index }); } if morpheme.span.start < previous_end { return Err(TokenizerError::InvalidUnitMetadata { index, reason: "morpheme spans overlap or are out of order", }); } if morpheme.surface.is_empty() != morpheme.span.is_empty() { return Err(TokenizerError::InvalidUnitMetadata { index, reason: "morpheme surface and span emptiness differ", }); } previous_end = previous_end.max(morpheme.span.end); } Ok(()) } /// Self-contained tokenizer using validated native assets. pub struct Tokenizer<'a> { morphology: NativeMorphology<'a>, disambiguator: NativeDisambiguator, config: TokenizerConfig, #[cfg(feature = "compiled-surface-table")] compiled_surface_analysis_table: Option, #[cfg(feature = "compiled-surface-table")] nedoformer_compiled_surface_analysis_table: Option, } #[derive(Debug)] struct RichAnalysisSet { analyses: Box<[NativeAnalysis]>, ambiguity: Box<[AmbiguityWordData]>, } struct AnalysisCache { entries: HashMap>, capacity: usize, } impl AnalysisCache { fn new(capacity: usize) -> Self { Self { entries: HashMap::with_capacity(capacity.min(4_096)), capacity, } } fn analyze( &mut self, morphology: &NativeMorphology<'_>, token: &str, ) -> Result, TokenizerError> { if let Some(analyses) = self.entries.get(token) { return Ok(Arc::clone(analyses)); } let analyses = surface_valid_candidates( token, 0, analyze_token_with_quality_fallback(morphology, token)?, ); let ambiguity = analyses.iter().map(ambiguity_word_data).collect::>(); let set = Arc::new(RichAnalysisSet { analyses: analyses.into_boxed_slice(), ambiguity: ambiguity.into_boxed_slice(), }); if self.entries.len() < self.capacity { self.entries.insert(token.to_owned(), Arc::clone(&set)); } Ok(set) } fn analyze_nedoformer( &mut self, morphology: &NativeMorphology<'_>, token: &str, ) -> Result, TokenizerError> { if let Some(cached) = self.entries.get(token) { return Ok(Arc::clone(cached)); } let analyses = surface_valid_candidates( token, 0, analyze_token_with_nedoformer_shadow(morphology, token)?, ); let ambiguity = analyses.iter().map(ambiguity_word_data).collect::>(); let set = Arc::new(RichAnalysisSet { analyses: analyses.into_boxed_slice(), ambiguity: ambiguity.into_boxed_slice(), }); if self.entries.len() < self.capacity { self.entries.insert(token.to_owned(), Arc::clone(&set)); } Ok(set) } } #[derive(Debug)] struct FlatAnalysisSet { relative_cuts: Box<[Vec]>, ambiguity: Box<[AmbiguityWordData]>, scoring_codes: Box<[AmbiguityScoringCode]>, unknown: Box<[bool]>, output_invariant: bool, } #[cfg(feature = "compiled-surface-table")] enum FlatAnalysisSource<'a> { Compiled(&'a FlatAnalysisSet), Live(Arc), } #[cfg(feature = "compiled-surface-table")] impl FlatAnalysisSource<'_> { fn set(&self) -> &FlatAnalysisSet { match self { Self::Compiled(set) => set, Self::Live(set) => set.as_ref(), } } } #[derive(Clone, Copy, Debug)] struct FlatSegmentTokenProgram { cut_start: u32, cut_len: u16, unknown: bool, } #[derive(Debug)] pub(crate) struct FlatSegmentProgram { tokens: Box<[FlatSegmentTokenProgram]>, relative_cuts: Box<[u32]>, pub(crate) surface_ids: Box<[u16]>, pub(crate) surface_lengths: Box<[u8]>, } #[derive(Debug)] struct FlatSurfaceProgramEntry { fingerprint: u64, kind: u8, exact: Box<[u8]>, program: Arc, } struct FlatSurfaceProgramTable { slots: Box<[Option]>, } impl FlatSurfaceProgramTable { fn new() -> Self { Self { slots: std::iter::repeat_with(|| None) .take(SURFACE_PROGRAM_TABLE_SLOTS) .collect::>() .into_boxed_slice(), } } fn clear(&mut self) { self.slots.fill_with(|| None); } } struct FlatDocumentProgramEntry { fingerprint: u64, newline: bool, exact: Box<[u8]>, ids: Box<[u16]>, lengths: Box<[u8]>, } struct FlatDocumentProgramTable { slots: Box<[Option]>, } impl FlatDocumentProgramTable { fn new(capacity: usize) -> Self { let slots = if capacity == 0 { 0 } else { capacity .saturating_mul(2) .max(8) .checked_next_power_of_two() .unwrap_or(usize::MAX / 2 + 1) }; Self { slots: std::iter::repeat_with(|| None) .take(slots) .collect::>() .into_boxed_slice(), } } fn clear(&mut self) { self.slots.fill_with(|| None); } } /// Aggregate telemetry for worker-local flat morphology caches. #[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] pub struct TrainingCacheStats { /// Successful surface lookups. pub hits: u64, /// Morphology calls caused by absent surfaces. pub misses: u64, /// Misses that could not be retained because a worker cache was full. pub saturated_misses: u64, /// Currently retained unique surfaces across workers. pub entries: u64, /// Approximate owned heap bytes for keys and compact candidate templates. pub approximate_bytes: u64, } impl TrainingCacheStats { const fn merge(&mut self, other: Self) { self.hits = self.hits.saturating_add(other.hits); self.misses = self.misses.saturating_add(other.misses); self.saturated_misses = self.saturated_misses.saturating_add(other.saturated_misses); self.entries = self.entries.saturating_add(other.entries); self.approximate_bytes = self .approximate_bytes .saturating_add(other.approximate_bytes); } } #[derive(Clone, Debug, Default)] struct AmbiguityScoringInterner { canonicals: HashMap, lemmas: HashMap, igs: HashMap, ig_sequences: HashMap, u32>, signatures: HashMap<(u32, u32), u32>, } impl AmbiguityScoringInterner { fn code(&mut self, word: &AmbiguityWordData) -> Result { let canonical = intern_scoring_string( &mut self.canonicals, &word.canonical, "canonical-analysis identity", )?; let lemma = intern_scoring_string(&mut self.lemmas, &word.lemma, "lemma identity")?; let mut ig_ids = Vec::with_capacity(word.igs.len()); for ig in &word.igs { ig_ids.push(intern_scoring_string( &mut self.igs, ig, "inflectional-group identity", )?); } let next_sequence = u32::try_from(self.ig_sequences.len()) .map_err(|_| TokenizerError::LengthOverflow("IG-sequence identity"))?; let ig_sequence = *self.ig_sequences.entry(ig_ids).or_insert(next_sequence); let next_signature = u32::try_from(self.signatures.len()) .map_err(|_| TokenizerError::LengthOverflow("ambiguity scoring signature"))?; let signature = *self .signatures .entry((lemma, ig_sequence)) .or_insert(next_signature); Ok(AmbiguityScoringCode::new( signature, canonical, word.java_hash, )) } fn signature_count(&self) -> usize { self.signatures.len() } fn code_with_base( &mut self, base: &Self, word: &AmbiguityWordData, ) -> Result { let canonical = intern_scoring_string_layered( &base.canonicals, &mut self.canonicals, &word.canonical, "canonical-analysis identity", )?; let lemma = intern_scoring_string_layered( &base.lemmas, &mut self.lemmas, &word.lemma, "lemma identity", )?; let mut ig_ids = Vec::with_capacity(word.igs.len()); for ig in &word.igs { ig_ids.push(intern_scoring_string_layered( &base.igs, &mut self.igs, ig, "inflectional-group identity", )?); } let ig_sequence = if let Some(&id) = base.ig_sequences.get(&ig_ids) { id } else if let Some(&id) = self.ig_sequences.get(&ig_ids) { id } else { let next = base .ig_sequences .len() .checked_add(self.ig_sequences.len()) .ok_or(TokenizerError::LengthOverflow("IG-sequence identity"))?; let id = u32::try_from(next) .map_err(|_| TokenizerError::LengthOverflow("IG-sequence identity"))?; self.ig_sequences.insert(ig_ids, id); id }; let signature_key = (lemma, ig_sequence); let signature = if let Some(&id) = base.signatures.get(&signature_key) { id } else if let Some(&id) = self.signatures.get(&signature_key) { id } else { let next = base .signatures .len() .checked_add(self.signatures.len()) .ok_or(TokenizerError::LengthOverflow( "ambiguity scoring signature", ))?; let id = u32::try_from(next) .map_err(|_| TokenizerError::LengthOverflow("ambiguity scoring signature"))?; self.signatures.insert(signature_key, id); id }; Ok(AmbiguityScoringCode::new( signature, canonical, word.java_hash, )) } fn approximate_bytes(&self) -> u64 { let mut bytes = self .canonicals .capacity() .saturating_mul(std::mem::size_of::<(String, u32)>()) .saturating_add( self.lemmas .capacity() .saturating_mul(std::mem::size_of::<(String, u32)>()), ) .saturating_add( self.igs .capacity() .saturating_mul(std::mem::size_of::<(String, u32)>()), ) .saturating_add( self.ig_sequences .capacity() .saturating_mul(std::mem::size_of::<(Vec, u32)>()), ) .saturating_add( self.signatures .capacity() .saturating_mul(std::mem::size_of::<((u32, u32), u32)>()), ); for value in self .canonicals .keys() .chain(self.lemmas.keys()) .chain(self.igs.keys()) { bytes = bytes.saturating_add(value.capacity()); } for value in self.ig_sequences.keys() { bytes = bytes.saturating_add(value.capacity().saturating_mul(std::mem::size_of::())); } u64::try_from(bytes).unwrap_or(u64::MAX) } } fn intern_scoring_string( values: &mut HashMap, value: &str, field: &'static str, ) -> Result { if let Some(&id) = values.get(value) { return Ok(id); } let id = u32::try_from(values.len()).map_err(|_| TokenizerError::LengthOverflow(field))?; values.insert(value.to_owned(), id); Ok(id) } fn intern_scoring_string_layered( base: &HashMap, delta: &mut HashMap, value: &str, field: &'static str, ) -> Result { if let Some(&id) = base.get(value) { return Ok(id); } if let Some(&id) = delta.get(value) { return Ok(id); } let next = base .len() .checked_add(delta.len()) .ok_or(TokenizerError::LengthOverflow(field))?; let id = u32::try_from(next).map_err(|_| TokenizerError::LengthOverflow(field))?; delta.insert(value.to_owned(), id); Ok(id) } const SHORT_ANALYSIS_KEY_BYTES: usize = 15; struct ShortAnalysisCache { keys: Vec, values: Vec>>, slots: usize, } impl ShortAnalysisCache { fn new(capacity: usize) -> Self { let slots = capacity .saturating_mul(2) .max(8) .checked_next_power_of_two() .unwrap_or(usize::MAX / 2 + 1); Self { keys: Vec::new(), values: Vec::new(), slots, } } #[inline(always)] fn get(&self, key: u128) -> Option<&Arc> { if self.keys.is_empty() { return None; } let modulo = self.keys.len() - 1; let mut slot = short_analysis_hash(key) & modulo; loop { let stored = self.keys[slot]; if stored == key { return self.values[slot].as_ref(); } if stored == 0 { return None; } slot = (slot + 1) & modulo; } } fn insert(&mut self, key: u128, value: Arc) { if self.keys.is_empty() { self.keys = vec![0; self.slots]; self.values = vec![None; self.slots]; } let modulo = self.keys.len() - 1; let mut slot = short_analysis_hash(key) & modulo; loop { if self.keys[slot] == 0 || self.keys[slot] == key { self.keys[slot] = key; self.values[slot] = Some(value); return; } slot = (slot + 1) & modulo; } } fn clear(&mut self) { self.keys = Vec::new(); self.values = Vec::new(); } fn approximate_bytes(&self) -> u64 { let bytes = self .keys .capacity() .saturating_mul(std::mem::size_of::()) .saturating_add( self.values .capacity() .saturating_mul(std::mem::size_of::>>()), ); u64::try_from(bytes).unwrap_or(u64::MAX) } } #[inline(always)] fn pack_short_analysis_key(token: &str) -> Option { let bytes = token.as_bytes(); if bytes.is_empty() || bytes.len() > SHORT_ANALYSIS_KEY_BYTES { return None; } let mut key = (bytes.len() as u128) << 120; for (index, byte) in bytes.iter().enumerate() { key |= u128::from(*byte) << (index * 8); } Some(key) } #[inline(always)] fn short_analysis_hash(key: u128) -> usize { let mut value = (key as u64) ^ ((key >> 64) as u64).rotate_left(23); value ^= value >> 30; value = value.wrapping_mul(0xbf58_476d_1ce4_e5b9); value ^= value >> 27; value = value.wrapping_mul(0x94d0_49bb_1331_11eb); value ^= value >> 31; usize::try_from(value).unwrap_or(0) } struct FlatAnalysisCache { entries: HashMap>, short_entries: ShortAnalysisCache, capacity: usize, entry_count: usize, hits: u64, misses: u64, saturated_misses: u64, approximate_bytes: u64, scoring_interner: AmbiguityScoringInterner, #[cfg(feature = "compiled-surface-table")] scoring_interner_seed: Option>, disambiguation_scores: DisambiguationScoreCache, segment_programs: HashMap, Arc>, segment_key: Vec, surface_programs: FlatSurfaceProgramTable, surface_program_entries: usize, surface_program_bytes: u64, document_programs: FlatDocumentProgramTable, document_program_capacity: usize, document_program_entries: usize, document_program_hits: u64, document_program_misses: u64, document_program_bytes: u64, surface_program_context: Option<(usize, bool)>, segment_program_hits: u64, segment_program_misses: u64, segment_program_bytes: u64, phase_scan_ns: u128, phase_code_ns: u128, phase_split_ns: u128, phase_analysis_ns: u128, phase_long_ns: u128, phase_vocab_ns: u128, phase_documents: u64, } fn estimate_flat_entry_bytes(token: &str, set: &FlatAnalysisSet) -> u64 { let mut bytes = token.len(); for cuts in &set.relative_cuts { bytes = bytes.saturating_add(cuts.len().saturating_mul(std::mem::size_of::())); } for ambiguity in &set.ambiguity { bytes = bytes .saturating_add(ambiguity.canonical.len()) .saturating_add(ambiguity.lemma.len()); for group in &ambiguity.igs { bytes = bytes.saturating_add(group.len()); } } bytes = bytes .saturating_add( set.scoring_codes .len() .saturating_mul(std::mem::size_of::()), ) .saturating_add(set.unknown.len()); u64::try_from(bytes).map_or(u64::MAX, |value| value) } impl FlatAnalysisCache { fn new( capacity: usize, document_program_capacity: usize, context_cache_policy: ContextCachePolicy, ) -> Self { Self { entries: HashMap::with_capacity(capacity.min(4_096)), short_entries: ShortAnalysisCache::new(capacity), capacity, entry_count: 0, hits: 0, misses: 0, saturated_misses: 0, approximate_bytes: 0, scoring_interner: AmbiguityScoringInterner::default(), #[cfg(feature = "compiled-surface-table")] scoring_interner_seed: None, disambiguation_scores: match context_cache_policy { ContextCachePolicy::Full => DisambiguationScoreCache::default(), ContextCachePolicy::Balanced => DisambiguationScoreCache::balanced(), ContextCachePolicy::Compact => DisambiguationScoreCache::compact(), }, segment_programs: HashMap::with_capacity(4_096), segment_key: Vec::with_capacity(512), surface_programs: FlatSurfaceProgramTable::new(), surface_program_entries: 0, surface_program_bytes: 0, document_programs: FlatDocumentProgramTable::new(document_program_capacity), document_program_capacity, document_program_entries: 0, document_program_hits: 0, document_program_misses: 0, document_program_bytes: 0, surface_program_context: None, segment_program_hits: 0, segment_program_misses: 0, segment_program_bytes: 0, phase_scan_ns: 0, phase_code_ns: 0, phase_split_ns: 0, phase_analysis_ns: 0, phase_long_ns: 0, phase_vocab_ns: 0, phase_documents: 0, } } #[cfg(feature = "compiled-surface-table")] fn new_seeded( capacity: usize, document_program_capacity: usize, context_cache_policy: ContextCachePolicy, scoring_interner_seed: Arc, shared_prepared_words: Option>, ) -> Self { let mut disambiguation_scores = match context_cache_policy { ContextCachePolicy::Full => DisambiguationScoreCache::default(), ContextCachePolicy::Balanced => DisambiguationScoreCache::balanced(), ContextCachePolicy::Compact => DisambiguationScoreCache::compact(), }; if let Some(shared_prepared_words) = shared_prepared_words { disambiguation_scores.set_shared_prepared_words(shared_prepared_words); } Self { entries: HashMap::with_capacity(capacity.min(4_096)), short_entries: ShortAnalysisCache::new(capacity), capacity, entry_count: 0, hits: 0, misses: 0, saturated_misses: 0, approximate_bytes: 0, scoring_interner: AmbiguityScoringInterner::default(), scoring_interner_seed: Some(scoring_interner_seed), disambiguation_scores, segment_programs: HashMap::with_capacity(4_096), segment_key: Vec::with_capacity(512), surface_programs: FlatSurfaceProgramTable::new(), surface_program_entries: 0, surface_program_bytes: 0, document_programs: FlatDocumentProgramTable::new(document_program_capacity), document_program_capacity, document_program_entries: 0, document_program_hits: 0, document_program_misses: 0, document_program_bytes: 0, surface_program_context: None, segment_program_hits: 0, segment_program_misses: 0, segment_program_bytes: 0, phase_scan_ns: 0, phase_code_ns: 0, phase_split_ns: 0, phase_analysis_ns: 0, phase_long_ns: 0, phase_vocab_ns: 0, phase_documents: 0, } } fn scoring_code( &mut self, word: &AmbiguityWordData, ) -> Result { #[cfg(feature = "compiled-surface-table")] if let Some(base) = self.scoring_interner_seed.as_deref() { return self.scoring_interner.code_with_base(base, word); } self.scoring_interner.code(word) } fn scoring_interner_counts(&self) -> (usize, usize, usize, usize) { #[cfg(feature = "compiled-surface-table")] let base = self.scoring_interner_seed.as_deref(); #[cfg(not(feature = "compiled-surface-table"))] let base: Option<&AmbiguityScoringInterner> = None; let base_signatures = base.map_or(0, |value| value.signatures.len()); let base_lemmas = base.map_or(0, |value| value.lemmas.len()); let base_igs = base.map_or(0, |value| value.igs.len()); let base_sequences = base.map_or(0, |value| value.ig_sequences.len()); ( base_signatures.saturating_add(self.scoring_interner.signatures.len()), base_lemmas.saturating_add(self.scoring_interner.lemmas.len()), base_igs.saturating_add(self.scoring_interner.igs.len()), base_sequences.saturating_add(self.scoring_interner.ig_sequences.len()), ) } fn analyze( &mut self, morphology: &NativeMorphology<'_>, token: &str, ) -> Result, TokenizerError> { let short_key = pack_short_analysis_key(token); if let Some(key) = short_key { if let Some(analyses) = self.short_entries.get(key) { self.hits = self.hits.saturating_add(1); return Ok(Arc::clone(analyses)); } } else if let Some(analyses) = self.entries.get(token) { self.hits = self.hits.saturating_add(1); return Ok(Arc::clone(analyses)); } self.misses = self.misses.saturating_add(1); let analyses = analyze_token_with_quality_fallback(morphology, token)?; let mut compact = build_flat_analysis_set(token, analyses)?; compact.scoring_codes = compact .ambiguity .iter() .map(|word| self.scoring_code(word)) .collect::, _>>()? .into_boxed_slice(); let compact = Arc::new(compact); if self.entry_count < self.capacity { self.approximate_bytes = self .approximate_bytes .saturating_add(estimate_flat_entry_bytes(token, &compact)); if let Some(key) = short_key { self.short_entries.insert(key, Arc::clone(&compact)); } else { self.entries.insert(token.to_owned(), Arc::clone(&compact)); } self.entry_count += 1; } else { self.saturated_misses = self.saturated_misses.saturating_add(1); } Ok(compact) } fn analyze_nedoformer( &mut self, morphology: &NativeMorphology<'_>, token: &str, ) -> Result, TokenizerError> { let short_key = pack_short_analysis_key(token); if let Some(key) = short_key { if let Some(analyses) = self.short_entries.get(key) { self.hits = self.hits.saturating_add(1); return Ok(Arc::clone(analyses)); } } else if let Some(analyses) = self.entries.get(token) { self.hits = self.hits.saturating_add(1); return Ok(Arc::clone(analyses)); } self.misses = self.misses.saturating_add(1); let analyses = analyze_token_with_nedoformer_shadow(morphology, token)?; let mut compact = build_flat_analysis_set(token, analyses)?; compact.scoring_codes = compact .ambiguity .iter() .map(|word| self.scoring_code(word)) .collect::, _>>()? .into_boxed_slice(); let compact = Arc::new(compact); if self.entry_count < self.capacity { self.approximate_bytes = self .approximate_bytes .saturating_add(estimate_flat_entry_bytes(token, &compact)); if let Some(key) = short_key { self.short_entries.insert(key, Arc::clone(&compact)); } else { self.entries.insert(token.to_owned(), Arc::clone(&compact)); } self.entry_count += 1; } else { self.saturated_misses = self.saturated_misses.saturating_add(1); } Ok(compact) } fn segment_program( &mut self, raw: &[u8], units: &[TokenizedUnit], indices: &[usize], ) -> Result>, TokenizerError> { self.segment_key.clear(); self.segment_key.extend_from_slice( &u32::try_from(indices.len()) .map_err(|_| TokenizerError::LengthOverflow("segment program token count"))? .to_le_bytes(), ); for index in indices { let unit = units .get(*index) .ok_or(TokenizerError::InvalidTrainingEncoding( "segment program unit index is out of range", ))?; self.segment_key.push(unit.kind as u8); let bytes = unit_bytes(raw, unit.span)?; self.segment_key.extend_from_slice( &u32::try_from(bytes.len()) .map_err(|_| TokenizerError::LengthOverflow("segment program surface length"))? .to_le_bytes(), ); self.segment_key.extend_from_slice(bytes); } if let Some(program) = self.segment_programs.get(self.segment_key.as_slice()) { self.segment_program_hits = self.segment_program_hits.saturating_add(1); self.hits = self .hits .saturating_add(u64::try_from(indices.len()).unwrap_or(u64::MAX)); return Ok(Some(Arc::clone(program))); } self.segment_program_misses = self.segment_program_misses.saturating_add(1); Ok(None) } fn insert_segment_program(&mut self, program: FlatSegmentProgram) { if self.segment_programs.len() >= SEGMENT_PROGRAM_CACHE_ENTRIES { return; } let bytes = self .segment_key .len() .saturating_add( program .tokens .len() .saturating_mul(std::mem::size_of::()), ) .saturating_add( program .relative_cuts .len() .saturating_mul(std::mem::size_of::()), ) .saturating_add( program .surface_ids .len() .saturating_mul(std::mem::size_of::()), ) .saturating_add(program.surface_lengths.len()); self.segment_program_bytes = self .segment_program_bytes .saturating_add(u64::try_from(bytes).unwrap_or(u64::MAX)); self.segment_programs .insert(self.segment_key.clone(), Arc::new(program)); } fn prepare_surface_program_context( &mut self, vocabulary: &SurfaceVocabulary, use_morphology: bool, ) { let context = ( vocabulary as *const SurfaceVocabulary as usize, use_morphology, ); if self.surface_program_context == Some(context) { return; } self.surface_programs.clear(); self.surface_program_entries = 0; self.surface_program_bytes = 0; self.document_programs.clear(); self.document_program_entries = 0; self.document_program_hits = 0; self.document_program_misses = 0; self.document_program_bytes = 0; self.surface_program_context = Some(context); } fn stats(&self) -> TrainingCacheStats { if std::env::var_os("NEDO_SEGMENT_PROGRAM_TELEMETRY").is_some() { eprintln!( "NEDO_SEGMENT_PROGRAM hits={} misses={} entries={} bytes={}", self.segment_program_hits, self.segment_program_misses, self.segment_programs .len() .saturating_add(self.surface_program_entries), self.segment_program_bytes .saturating_add(self.surface_program_bytes) .saturating_add(self.document_program_bytes), ); } if std::env::var_os("NEDO_SURFACE_PHASE_TELEMETRY").is_some() { let total = self .phase_scan_ns .saturating_add(self.phase_code_ns) .saturating_add(self.phase_split_ns) .saturating_add(self.phase_analysis_ns) .saturating_add(self.phase_long_ns) .saturating_add(self.phase_vocab_ns); eprintln!( "NEDO_SURFACE_PHASE documents={} total_ns={} scan_ns={} code_ns={} split_ns={} analysis_ns={} long_ns={} vocab_ns={}", self.phase_documents, total, self.phase_scan_ns, self.phase_code_ns, self.phase_split_ns, self.phase_analysis_ns, self.phase_long_ns, self.phase_vocab_ns, ); } if std::env::var_os("NEDO_TRIGRAM_SCORE_CACHE_TELEMETRY").is_some() { let (hits, misses, entries, slots) = self.disambiguation_scores.stats(); let (rows, row_capacity, pair_slots, overflow_entries, overflow_slots) = self.disambiguation_scores.layout_stats(); let ( dense_attempts, dense_successes, dense_duplicate_fallbacks, dense_state_fallbacks, dense_tie_fallbacks, ) = self.disambiguation_scores.dense_stats(); eprintln!( "NEDO_TRIGRAM_SCORE_CACHE hits={hits} misses={misses} entries={entries} slots={slots} rows={rows} row_capacity={row_capacity} pair_slots={pair_slots} overflow_entries={overflow_entries} overflow_slots={overflow_slots} dense_attempts={dense_attempts} dense_successes={dense_successes} dense_duplicate_fallbacks={dense_duplicate_fallbacks} dense_state_fallbacks={dense_state_fallbacks} dense_tie_fallbacks={dense_tie_fallbacks} signatures={} lemmas={} igs={} ig_sequences={}", self.scoring_interner_counts().0, self.scoring_interner_counts().1, self.scoring_interner_counts().2, self.scoring_interner_counts().3, ); } if std::env::var_os("NEDO_DOCUMENT_PROGRAM_TELEMETRY").is_some() { eprintln!( "NEDO_DOCUMENT_PROGRAM hits={} misses={} entries={} bytes={}", self.document_program_hits, self.document_program_misses, self.document_program_entries, self.document_program_bytes, ); } TrainingCacheStats { hits: self.hits, misses: self.misses, saturated_misses: self.saturated_misses, entries: u64::try_from(self.entry_count).map_or(u64::MAX, |value| value), approximate_bytes: self .approximate_bytes .saturating_add(self.scoring_interner.approximate_bytes()) .saturating_add(self.disambiguation_scores.approximate_bytes()) .saturating_add(self.short_entries.approximate_bytes()) .saturating_add(self.segment_program_bytes) .saturating_add(self.surface_program_bytes) .saturating_add(self.document_program_bytes), } } fn clear(&mut self) { self.entries.clear(); self.short_entries.clear(); self.entry_count = 0; self.hits = 0; self.misses = 0; self.saturated_misses = 0; self.approximate_bytes = 0; #[cfg(feature = "compiled-surface-table")] { self.scoring_interner = AmbiguityScoringInterner::default(); } #[cfg(not(feature = "compiled-surface-table"))] { self.scoring_interner = AmbiguityScoringInterner::default(); } self.disambiguation_scores.clear(); self.segment_programs.clear(); self.segment_key.clear(); self.surface_programs.clear(); self.surface_program_entries = 0; self.surface_program_bytes = 0; self.document_programs.clear(); self.document_program_entries = 0; self.document_program_hits = 0; self.document_program_misses = 0; self.document_program_bytes = 0; self.surface_program_context = None; self.segment_program_hits = 0; self.segment_program_misses = 0; self.segment_program_bytes = 0; self.phase_scan_ns = 0; self.phase_code_ns = 0; self.phase_split_ns = 0; self.phase_analysis_ns = 0; self.phase_long_ns = 0; self.phase_vocab_ns = 0; self.phase_documents = 0; } } /// Stateful metadata-free training encoder with persistent worker-local caches. pub struct FlatTrainingEncoder<'tokenizer, 'assets> { tokenizer: &'tokenizer Tokenizer<'assets>, vocabulary: &'tokenizer CharacterVocabulary, threads: usize, options: TrainingEncodingOptions, caches: Vec, } impl FlatTrainingEncoder<'_, '_> { /// Encodes one batch while preserving worker-local caches across calls. /// /// # Errors /// /// Returns an error for input/newline cardinality mismatch, worker failure, /// or any tokenizer/training-stream invariant violation. pub fn encode_batch( &mut self, inputs: &[Vec], newline_flags: &[bool], ) -> Result { if inputs.len() != newline_flags.len() { return Err(TokenizerError::InvalidTrainingEncoding( "flat input and newline counts differ", )); } if inputs.is_empty() { return Ok(TrainingBatch { ids: Vec::new(), lengths: Vec::new(), document_offsets: vec![0], }); } let ranges = build_batch_ranges(inputs, self.threads)?; let worker_count = self.threads.min(ranges.len()); let chunks = if worker_count == 1 { let cache = self .caches .first_mut() .ok_or(TokenizerError::InvalidConfiguration( "flat encoder has no worker cache", ))?; let mut encoded = Vec::with_capacity(inputs.len()); for (raw, newline) in inputs.iter().zip(newline_flags) { encoded.push(self.tokenizer.encode_flat_document( raw.clone(), *newline, self.vocabulary, self.options, cache, )?); } vec![(0, encoded)] } else { let next = AtomicUsize::new(0); let tokenizer = self.tokenizer; let vocabulary = self.vocabulary; let options = self.options; let worker_chunks = thread::scope(|scope| { let handles = self.caches[..worker_count] .iter_mut() .map(|cache| { let ranges = &ranges; let next = &next; scope.spawn(move || { let mut completed = Vec::new(); loop { let chunk_index = next.fetch_add(1, Ordering::Relaxed); let Some(range) = ranges.get(chunk_index) else { break; }; let mut encoded = Vec::with_capacity(range.len()); for index in range.clone() { encoded.push(tokenizer.encode_flat_document( inputs[index].clone(), newline_flags[index], vocabulary, options, cache, )?); } completed.push((chunk_index, encoded)); } Ok::<_, TokenizerError>(completed) }) }) .collect::>(); handles .into_iter() .map(|handle| handle.join().map_err(|_| TokenizerError::WorkerPanicked)?) .collect::, _>>() })?; worker_chunks.into_iter().flatten().collect::>() }; concatenate_training_chunks(chunks) } /// Returns aggregate cache telemetry across workers. #[must_use] pub fn cache_stats(&self) -> TrainingCacheStats { let mut stats = TrainingCacheStats::default(); for cache in &self.caches { stats.merge(cache.stats()); } stats } /// Clears all worker-local cache entries and counters. pub fn clear_caches(&mut self) { for cache in &mut self.caches { cache.clear(); } } } fn encode_flat_surface_batch_with_caches( tokenizer: &Tokenizer<'_>, inputs: &[Vec], newline_flags: &[bool], vocabulary: &SurfaceVocabulary, threads: usize, use_morphology: bool, caches: &mut [FlatAnalysisCache], ) -> Result { if inputs.len() != newline_flags.len() { return Err(TokenizerError::InvalidTrainingEncoding( "flat surface input and newline counts differ", )); } if inputs.is_empty() { return Ok(TrainingBatch { ids: Vec::new(), lengths: Vec::new(), document_offsets: vec![0], }); } if caches.len() < threads { return Err(TokenizerError::InvalidConfiguration( "surface runtime has fewer worker caches than threads", )); } let ranges = build_batch_ranges(inputs, threads)?; let worker_count = threads.min(ranges.len()); if worker_count == 1 { let cache = caches .first_mut() .ok_or(TokenizerError::InvalidConfiguration( "surface runtime has no worker cache", ))?; return encode_flat_surface_range( tokenizer, inputs, newline_flags, 0..inputs.len(), vocabulary, use_morphology, cache, ); } let next = AtomicUsize::new(0); let worker_chunks = thread::scope(|scope| { let handles = caches[..worker_count] .iter_mut() .map(|cache| { let ranges = &ranges; let next = &next; scope.spawn(move || { let mut completed = Vec::new(); loop { let chunk_index = next.fetch_add(1, Ordering::Relaxed); let Some(range) = ranges.get(chunk_index) else { break; }; completed.push(( chunk_index, encode_flat_surface_range( tokenizer, inputs, newline_flags, range.clone(), vocabulary, use_morphology, cache, )?, )); } Ok::<_, TokenizerError>(completed) }) }) .collect::>(); handles .into_iter() .map(|handle| handle.join().map_err(|_| TokenizerError::WorkerPanicked)?) .collect::, _>>() })?; concatenate_surface_chunks(worker_chunks.into_iter().flatten().collect()) } /// Opaque persistent worker-local runtime for repeated surface batches. /// /// The runtime is bound to the exact tokenizer configuration and compiled-table /// payload identity that created it. Passing it to a different tokenizer is /// rejected before any document is encoded. pub struct SurfaceRuntimeCache { config: TokenizerConfig, compiled_table_digest: Option<[u8; 32]>, threads: usize, options: SurfaceEncoderOptions, caches: Vec, } impl SurfaceRuntimeCache { /// Number of worker caches retained by this runtime. #[must_use] pub const fn threads(&self) -> usize { self.threads } /// Returns aggregate persistent-cache telemetry. #[must_use] pub fn cache_stats(&self) -> TrainingCacheStats { let mut stats = TrainingCacheStats::default(); for cache in &self.caches { stats.merge(cache.stats()); } stats } /// Clears every worker-local cache and counter without changing runtime policy. pub fn clear(&mut self) { for cache in &mut self.caches { cache.clear(); } } } /// Stateful metadata-free surface-piece encoder with persistent worker-local caches. pub struct FlatSurfaceEncoder<'tokenizer, 'assets> { tokenizer: &'tokenizer Tokenizer<'assets>, vocabulary: &'tokenizer SurfaceVocabulary, runtime: SurfaceRuntimeCache, } impl FlatSurfaceEncoder<'_, '_> { /// Encodes one batch directly to final surface-piece IDs. /// /// # Errors /// /// Returns an error for input/newline cardinality mismatch, worker failure, /// morphology/disambiguation failure, or surface encoding failure. pub fn encode_batch( &mut self, inputs: &[Vec], newline_flags: &[bool], ) -> Result { self.tokenizer.encode_surface_batch_with_runtime( inputs, newline_flags, self.vocabulary, &mut self.runtime, ) } /// Returns aggregate cache telemetry across workers. #[must_use] pub fn cache_stats(&self) -> TrainingCacheStats { self.runtime.cache_stats() } /// Clears all worker-local cache entries and counters. pub fn clear_caches(&mut self) { self.runtime.clear(); } } impl Tokenizer<'static> { /// Creates the production tokenizer from embedded, checksum-pinned assets. /// /// # Errors /// /// Returns an error if embedded checksums or native asset validation fails. pub fn embedded(config: TokenizerConfig) -> Result { verify_sha(EMBEDDED_MORPHOLOGY, MORPHOLOGY_SHA256, "morphology")?; verify_sha(EMBEDDED_MODEL, MODEL_SHA256, "disambiguation model")?; Self::from_bytes(EMBEDDED_MORPHOLOGY, EMBEDDED_MODEL, config) } } impl<'a> Tokenizer<'a> { /// Creates a tokenizer only when supplied assets exactly match schema-v1 identities. /// /// # Errors /// /// Returns an error for invalid configuration, checksums, morphology, or model bytes. pub fn from_bytes( morphology: &'a [u8], model: &[u8], config: TokenizerConfig, ) -> Result { if config.max_sentence_tokens == 0 { return Err(TokenizerError::InvalidConfiguration( "max_sentence_tokens must be positive", )); } if config.max_fallback_chars == 0 || config.max_fallback_chars > MAX_FALLBACK_CHARS_CONFIG { return Err(TokenizerError::InvalidConfiguration( "max_fallback_chars must be in 1..=4096", )); } verify_sha(morphology, MORPHOLOGY_SHA256, "morphology")?; verify_sha(model, MODEL_SHA256, "disambiguation model")?; Ok(Self { morphology: NativeMorphology::parse(morphology)?, disambiguator: NativeDisambiguator::from_bytes(model)?, config, #[cfg(feature = "compiled-surface-table")] compiled_surface_analysis_table: None, #[cfg(feature = "compiled-surface-table")] nedoformer_compiled_surface_analysis_table: None, }) } /// Attaches a full compiled analysis table after exact semantic revalidation. /// /// Every surface and candidate is regenerated by the active native morphology. /// Candidate order, cuts, unknown status, canonical identity, lemma, IG sequence, /// and Java hash must all match before the table can enter the hot path. /// /// # Errors /// /// Returns an error on invalid UTF-8, morphology/alignment failure, or any /// candidate metadata mismatch. #[cfg(feature = "compiled-surface-table")] pub fn with_verified_compiled_surface_analysis_table( mut self, table: CompiledSurfaceAnalysisTable, ) -> Result { for (surface_bytes, compiled) in table.entries() { let surface = std::str::from_utf8(surface_bytes).map_err(|_| { TokenizerError::InvalidCompiledSurfaceTable { surface: hex_bytes(surface_bytes), reason: "surface key is not valid UTF-8", } })?; let exact = build_flat_analysis_set( surface, analyze_token_with_quality_fallback(&self.morphology, surface)?, )?; if exact.relative_cuts != compiled.relative_cuts { return Err(TokenizerError::InvalidCompiledSurfaceTable { surface: surface.to_owned(), reason: "candidate cuts or candidate order differ", }); } if exact.unknown != compiled.unknown { return Err(TokenizerError::InvalidCompiledSurfaceTable { surface: surface.to_owned(), reason: "candidate unknown status or order differs", }); } if exact.ambiguity != compiled.ambiguity { return Err(TokenizerError::InvalidCompiledSurfaceTable { surface: surface.to_owned(), reason: "candidate scoring metadata or order differs", }); } if compiled.scoring_codes.len() != compiled.ambiguity.len() { return Err(TokenizerError::InvalidCompiledSurfaceTable { surface: surface.to_owned(), reason: "compiled scoring-code cardinality differs", }); } } self.compiled_surface_analysis_table = Some(table); Ok(self) } /// Attaches an exact NedoFormer shadow-analysis lookup table. /// /// Unlike the standalone surface table, every entry is regenerated through /// the NedoFormer raw/lowercase/deasciify shadow analyzer before acceptance. /// This prevents a normal-surface table from silently changing lattice cuts. /// /// # Errors /// /// Returns an error on invalid UTF-8, shadow-analysis/alignment failure, or /// any candidate metadata mismatch. #[cfg(feature = "compiled-surface-table")] pub fn with_verified_nedoformer_compiled_surface_analysis_table( mut self, table: CompiledSurfaceAnalysisTable, ) -> Result { for (surface_bytes, compiled) in table.entries() { let surface = std::str::from_utf8(surface_bytes).map_err(|_| { TokenizerError::InvalidCompiledSurfaceTable { surface: hex_bytes(surface_bytes), reason: "NedoFormer surface key is not valid UTF-8", } })?; let exact = build_flat_analysis_set( surface, analyze_token_with_nedoformer_shadow(&self.morphology, surface)?, )?; if exact.relative_cuts != compiled.relative_cuts { return Err(TokenizerError::InvalidCompiledSurfaceTable { surface: surface.to_owned(), reason: "NedoFormer candidate cuts or order differ", }); } if exact.unknown != compiled.unknown { return Err(TokenizerError::InvalidCompiledSurfaceTable { surface: surface.to_owned(), reason: "NedoFormer candidate unknown status or order differs", }); } if exact.ambiguity != compiled.ambiguity { return Err(TokenizerError::InvalidCompiledSurfaceTable { surface: surface.to_owned(), reason: "NedoFormer candidate scoring metadata or order differs", }); } if compiled.scoring_codes.len() != compiled.ambiguity.len() { return Err(TokenizerError::InvalidCompiledSurfaceTable { surface: surface.to_owned(), reason: "NedoFormer compiled scoring-code cardinality differs", }); } } self.nedoformer_compiled_surface_analysis_table = Some(table); Ok(self) } /// Attaches a NedoFormer compiled table whose payload digest was pinned by a trusted manifest. /// /// [`CompiledSurfaceAnalysisTable::from_bytes`] has already validated the table schema, /// embedded morphology/model identities, payload checksum, UTF-8 surfaces, cuts, and /// candidate structure. This fast path additionally requires the payload digest to match /// an independently trusted value, allowing production startup to skip regenerating every /// table entry through morphology. The expected digest must come from a trusted release /// manifest or other out-of-band seal; deriving it from the same untrusted table defeats /// the purpose of this check. /// /// Use [`Self::with_verified_nedoformer_compiled_surface_analysis_table`] when establishing /// trust in a newly built table or when no trusted digest is available. /// /// # Errors /// /// Returns an error when the parsed table payload digest differs from `expected_digest`. #[cfg(feature = "compiled-surface-table")] pub fn with_pinned_nedoformer_compiled_surface_analysis_table( mut self, table: CompiledSurfaceAnalysisTable, expected_digest: [u8; 32], ) -> Result { let actual = table.digest(); if actual != expected_digest { return Err(TokenizerError::CompiledSurfaceTableSealMismatch { expected: hex_bytes(&expected_digest), actual: hex_bytes(&actual), }); } self.nedoformer_compiled_surface_analysis_table = Some(table); Ok(self) } /// Tokenizes one arbitrary byte document without normalization or replacement. /// /// # Errors /// /// Returns an error for scanning, morphology, context decoding, alignment, or invariant failure. pub fn tokenize(&self, raw: Vec) -> Result { let mut cache = AnalysisCache::new(0); self.tokenize_with_cache(raw, &mut cache) } fn tokenize_with_cache( &self, raw: Vec, cache: &mut AnalysisCache, ) -> Result { let scan = scan(raw)?; let raw = scan.document().decode(); let code_spans = match self.config.mode { TokenizerMode::Auto if self.config.detect_unmarked_code => { auto_code_spans(raw, scan.code_hints()) } TokenizerMode::Auto => explicit_code_spans(raw), TokenizerMode::Turkish | TokenizerMode::Code => Vec::new(), }; let mut units = split_units(&scan, &code_spans, self.config.mode)?; self.apply_contextual_analysis(raw, &mut units, cache)?; let mut units = split_long_fallback_units(raw, units, self.config.max_fallback_chars)?; assign_inner_groups(raw, &mut units)?; let (raw, _) = scan.into_document().into_parts(); TokenizedDocument::new(raw, units) } /// Deterministic native parallel batch tokenization. /// /// Work is split into contiguous chunks and results are reassembled in input /// order. No per-token language-boundary callback is used. /// /// # Errors /// /// Returns an error for zero threads, worker failure, or any document tokenization failure. pub fn tokenize_batch( &self, inputs: &[Vec], threads: usize, ) -> Result, TokenizerError> { if inputs.is_empty() { return Ok(Vec::new()); } if threads == 0 { return Err(TokenizerError::InvalidConfiguration( "batch thread count must be positive", )); } let ranges = build_batch_ranges(inputs, threads)?; let worker_count = threads.min(ranges.len()); if worker_count == 1 { let mut cache = AnalysisCache::new(BATCH_ANALYSIS_CACHE_ENTRIES); return inputs .iter() .cloned() .map(|raw| self.tokenize_with_cache(raw, &mut cache)) .collect(); } let next = AtomicUsize::new(0); let worker_chunks = thread::scope(|scope| { let handles = (0..worker_count) .map(|_| { let ranges = &ranges; let next = &next; scope.spawn(move || { let mut cache = AnalysisCache::new(BATCH_ANALYSIS_CACHE_ENTRIES); let mut completed = Vec::new(); loop { let chunk_index = next.fetch_add(1, Ordering::Relaxed); let Some(range) = ranges.get(chunk_index) else { break; }; let documents = inputs[range.clone()] .iter() .cloned() .map(|raw| self.tokenize_with_cache(raw, &mut cache)) .collect::, _>>()?; completed.push((chunk_index, documents)); } Ok::<_, TokenizerError>(completed) }) }) .collect::>(); handles .into_iter() .map(|handle| handle.join().map_err(|_| TokenizerError::WorkerPanicked)?) .collect::, _>>() })?; let mut chunks = worker_chunks.into_iter().flatten().collect::>(); chunks.sort_unstable_by_key(|entry| entry.0); Ok(chunks .into_iter() .flat_map(|(_, documents)| documents) .collect()) } /// Encodes a batch directly into flat training streams without building /// rich [`TokenizedDocument`] or [`AnalysisMetadata`] object graphs. /// /// Input order, structural controls, selected morphology, byte lengths, and /// document token offsets are exact with the rich reference encoder. /// /// # Errors /// /// Returns an error for invalid thread counts, cardinality mismatch, /// morphology/disambiguation failures, or any flat-stream invariant failure. pub fn encode_training_batch( &self, inputs: &[Vec], newline_flags: &[bool], vocabulary: &CharacterVocabulary, threads: usize, options: TrainingEncodingOptions, ) -> Result { let mut encoder = self.training_encoder(vocabulary, threads, options)?; encoder.encode_batch(inputs, newline_flags) } #[cfg_attr( not(feature = "compiled-surface-table"), allow(clippy::missing_const_for_fn, clippy::unused_self) )] fn compiled_surface_table_digest(&self) -> Option<[u8; 32]> { #[cfg(feature = "compiled-surface-table")] { self.compiled_surface_analysis_table .as_ref() .map(CompiledSurfaceAnalysisTable::digest) } #[cfg(not(feature = "compiled-surface-table"))] { None } } fn new_flat_analysis_cache( &self, capacity: usize, document_program_capacity: usize, context_cache_policy: ContextCachePolicy, ) -> FlatAnalysisCache { #[cfg(feature = "compiled-surface-table")] if let Some(table) = self.compiled_surface_analysis_table.as_ref() { return FlatAnalysisCache::new_seeded( capacity, document_program_capacity, context_cache_policy, table.scoring_interner().clone(), None, ); } FlatAnalysisCache::new(capacity, document_program_capacity, context_cache_policy) } #[cfg_attr( not(feature = "compiled-surface-table"), allow(clippy::missing_const_for_fn, clippy::unused_self) )] fn nedoformer_compiled_surface_table_digest(&self) -> Option<[u8; 32]> { #[cfg(feature = "compiled-surface-table")] { self.nedoformer_compiled_surface_analysis_table .as_ref() .map(CompiledSurfaceAnalysisTable::digest) } #[cfg(not(feature = "compiled-surface-table"))] { None } } fn new_nedoformer_flat_analysis_cache( &self, capacity: usize, context_cache_policy: ContextCachePolicy, shared_prepared_words: Option>, ) -> FlatAnalysisCache { #[cfg(feature = "compiled-surface-table")] if let Some(table) = self.nedoformer_compiled_surface_analysis_table.as_ref() { return FlatAnalysisCache::new_seeded( capacity, 0, context_cache_policy, table.scoring_interner().clone(), shared_prepared_words, ); } FlatAnalysisCache::new(capacity, 0, context_cache_policy) } #[cfg_attr( not(feature = "compiled-surface-table"), allow(clippy::missing_const_for_fn, clippy::unused_self) )] fn new_nedoformer_shared_prepared_word_cache(&self) -> Option> { #[cfg(feature = "compiled-surface-table")] { self.nedoformer_compiled_surface_analysis_table .as_ref() .map(|table| { Arc::new(SharedPreparedWordCache::with_slots( table.scoring_interner().signature_count(), )) }) } #[cfg(not(feature = "compiled-surface-table"))] { None } } /// Creates a stateful flat encoder whose worker-local caches survive calls. /// /// # Errors /// /// Returns an error when `threads` is zero. pub fn training_encoder<'tokenizer>( &'tokenizer self, vocabulary: &'tokenizer CharacterVocabulary, threads: usize, options: TrainingEncodingOptions, ) -> Result, TokenizerError> { if threads == 0 { return Err(TokenizerError::InvalidConfiguration( "flat batch thread count must be positive", )); } let analysis_cache_entries = analysis_cache_entries_for_parallelism(threads); Ok(FlatTrainingEncoder { tokenizer: self, vocabulary, threads, options, caches: (0..threads) .map(|_| { self.new_flat_analysis_cache( analysis_cache_entries, 0, ContextCachePolicy::Full, ) }) .collect(), }) } /// Creates an opaque persistent runtime for repeated surface batches. /// /// # Errors /// /// Returns an error when `threads` is zero. pub fn surface_runtime_cache( &self, threads: usize, options: SurfaceEncoderOptions, ) -> Result { if threads == 0 { return Err(TokenizerError::InvalidConfiguration( "surface runtime thread count must be positive", )); } self.surface_runtime_cache_with_analysis_entries( threads, options, analysis_cache_entries_for_parallelism(threads), ) } pub(crate) fn surface_runtime_cache_with_analysis_entries( &self, threads: usize, options: SurfaceEncoderOptions, analysis_cache_entries: usize, ) -> Result { if threads == 0 { return Err(TokenizerError::InvalidConfiguration( "surface runtime thread count must be positive", )); } Ok(SurfaceRuntimeCache { config: self.config, compiled_table_digest: self.compiled_surface_table_digest(), threads, options, caches: (0..threads) .map(|_| { self.new_flat_analysis_cache( analysis_cache_entries, options.document_cache_entries, options.context_cache_policy, ) }) .collect(), }) } /// Encodes with a persistent runtime created by [`Self::surface_runtime_cache`]. /// /// # Errors /// /// Rejects cardinality mismatch, a runtime from a different tokenizer/table, /// worker failures, and all normal encoding failures. pub fn encode_surface_batch_with_runtime( &self, inputs: &[Vec], newline_flags: &[bool], vocabulary: &SurfaceVocabulary, runtime: &mut SurfaceRuntimeCache, ) -> Result { if runtime.config != self.config || runtime.compiled_table_digest != self.compiled_surface_table_digest() { return Err(TokenizerError::InvalidConfiguration( "surface runtime belongs to a different tokenizer configuration or compiled table", )); } encode_flat_surface_batch_with_caches( self, inputs, newline_flags, vocabulary, runtime.threads, runtime.options.use_morphology, &mut runtime.caches, ) } /// Encodes one batch directly to final surface-piece IDs. /// /// # Errors /// /// Returns an error when the thread count is zero or encoding fails. pub fn encode_surface_batch( &self, inputs: &[Vec], newline_flags: &[bool], vocabulary: &SurfaceVocabulary, threads: usize, use_morphology: bool, ) -> Result { let mut encoder = self.surface_encoder(vocabulary, threads, use_morphology)?; encoder.encode_batch(inputs, newline_flags) } /// Creates a stateful final surface-piece encoder with persistent caches. /// /// # Errors /// /// Returns an error when `threads` is zero. pub fn surface_encoder<'tokenizer>( &'tokenizer self, vocabulary: &'tokenizer SurfaceVocabulary, threads: usize, use_morphology: bool, ) -> Result, TokenizerError> { self.surface_encoder_with_options( vocabulary, threads, SurfaceEncoderOptions::cached(use_morphology), ) } /// Creates a stateful final surface-piece encoder with an explicit cache policy. /// /// Use [`SurfaceEncoderOptions::one_pass`] for new-corpus preprocessing so /// complete source documents and their output vectors are not retained. /// /// # Errors /// /// Returns an error when `threads` is zero. pub fn surface_encoder_with_options<'tokenizer>( &'tokenizer self, vocabulary: &'tokenizer SurfaceVocabulary, threads: usize, options: SurfaceEncoderOptions, ) -> Result, TokenizerError> { if threads == 0 { return Err(TokenizerError::InvalidConfiguration( "flat surface batch thread count must be positive", )); } self.surface_encoder_with_analysis_cache_entries( vocabulary, threads, options, analysis_cache_entries_for_parallelism(threads), ) } pub(crate) fn surface_encoder_with_analysis_cache_entries<'tokenizer>( &'tokenizer self, vocabulary: &'tokenizer SurfaceVocabulary, threads: usize, options: SurfaceEncoderOptions, analysis_cache_entries: usize, ) -> Result, TokenizerError> { Ok(FlatSurfaceEncoder { tokenizer: self, vocabulary, runtime: self.surface_runtime_cache_with_analysis_entries( threads, options, analysis_cache_entries, )?, }) } fn encode_flat_document( &self, raw: Vec, newline: bool, vocabulary: &CharacterVocabulary, options: TrainingEncodingOptions, cache: &mut FlatAnalysisCache, ) -> Result { let scan = scan_compact(raw)?; let raw = scan.raw(); let code_spans = match self.config.mode { TokenizerMode::Auto if self.config.detect_unmarked_code => { auto_code_spans(raw, scan.code_hints()) } TokenizerMode::Auto => explicit_code_spans(raw), TokenizerMode::Turkish | TokenizerMode::Code => Vec::new(), }; let mut units = split_compact_units(&scan, &code_spans, self.config.mode)?; self.apply_flat_contextual_analysis(raw, &mut units, cache)?; let units = split_long_flat_units(raw, units, self.config.max_fallback_chars)?; vocabulary.encode_training_units(raw, &units, newline, options) } fn append_flat_surface_document( &self, raw: &[u8], newline: bool, vocabulary: &SurfaceVocabulary, use_morphology: bool, cache: &mut FlatAnalysisCache, output: &mut TrainingBatch, ) -> Result<(), TokenizerError> { cache.prepare_surface_program_context(vocabulary, use_morphology); if flat_surface::try_append_cached_document_program( &raw, newline, cache, &mut output.ids, &mut output.lengths, )? { output.document_offsets.push( u64::try_from(output.ids.len()) .map_err(|_| TokenizerError::LengthOverflow("surface token offset"))?, ); return Ok(()); } let id_start = output.ids.len(); let length_start = output.lengths.len(); let exact = (cache.document_program_entries < cache.document_program_capacity) .then(|| raw.to_vec().into_boxed_slice()); self.append_flat_surface_document_uncached( raw, newline, vocabulary, use_morphology, cache, output, )?; if let Some(exact) = exact { flat_surface::insert_document_program( cache, exact, newline, &output.ids[id_start..], &output.lengths[length_start..], )?; } Ok(()) } fn append_flat_surface_document_uncached( &self, raw: &[u8], newline: bool, vocabulary: &SurfaceVocabulary, use_morphology: bool, cache: &mut FlatAnalysisCache, output: &mut TrainingBatch, ) -> Result<(), TokenizerError> { let telemetry = std::env::var_os("NEDO_SURFACE_PHASE_TELEMETRY").is_some(); match self.config.mode { TokenizerMode::Auto => { let started = telemetry.then(std::time::Instant::now); if flat_surface::try_append_cached_auto_raw_document( raw, newline, vocabulary, self.config.max_sentence_tokens, self.config.max_fallback_chars, self.config.detect_unmarked_code, cache, &mut output.ids, &mut output.lengths, )? { if let Some(started) = started { cache.phase_scan_ns = cache .phase_scan_ns .saturating_add(started.elapsed().as_nanos()); cache.phase_documents = cache.phase_documents.saturating_add(1); } output.document_offsets.push( u64::try_from(output.ids.len()) .map_err(|_| TokenizerError::LengthOverflow("surface token offset"))?, ); return Ok(()); } let scan = scan_compact(raw.to_vec())?; if let Some(started) = started { cache.phase_scan_ns = cache .phase_scan_ns .saturating_add(started.elapsed().as_nanos()); } let raw = scan.raw(); let started = telemetry.then(std::time::Instant::now); let code_spans = if self.config.detect_unmarked_code { auto_code_spans(raw, scan.code_hints()) } else { explicit_code_spans(raw) }; if let Some(started) = started { cache.phase_code_ns = cache .phase_code_ns .saturating_add(started.elapsed().as_nanos()); } let started = telemetry.then(std::time::Instant::now); if code_spans.is_empty() && flat_surface::try_append_cached_auto_document( &scan, newline, vocabulary, self.config.max_sentence_tokens, self.config.max_fallback_chars, cache, &mut output.ids, &mut output.lengths, )? { if let Some(started) = started { cache.phase_split_ns = cache .phase_split_ns .saturating_add(started.elapsed().as_nanos()); cache.phase_documents = cache.phase_documents.saturating_add(1); } output.document_offsets.push( u64::try_from(output.ids.len()) .map_err(|_| TokenizerError::LengthOverflow("surface token offset"))?, ); return Ok(()); } let units = flat_surface::split_units(&scan, &code_spans, self.config.mode)?; if let Some(started) = started { cache.phase_split_ns = cache .phase_split_ns .saturating_add(started.elapsed().as_nanos()); } self.append_flat_surface_units( raw, units, newline, vocabulary, use_morphology, telemetry, cache, output, ) } TokenizerMode::Turkish => { let started = telemetry.then(std::time::Instant::now); if flat_surface::try_append_cached_turkish_document( raw, newline, vocabulary, self.config.max_sentence_tokens, self.config.max_fallback_chars, cache, &mut output.ids, &mut output.lengths, )? { if let Some(started) = started { cache.phase_scan_ns = cache .phase_scan_ns .saturating_add(started.elapsed().as_nanos()); cache.phase_documents = cache.phase_documents.saturating_add(1); } output.document_offsets.push( u64::try_from(output.ids.len()) .map_err(|_| TokenizerError::LengthOverflow("surface token offset"))?, ); return Ok(()); } let units = flat_surface::scan_fixed_units(raw, self.config.mode)?; if let Some(started) = started { cache.phase_scan_ns = cache .phase_scan_ns .saturating_add(started.elapsed().as_nanos()); } self.append_flat_surface_units( raw, units, newline, vocabulary, use_morphology, telemetry, cache, output, ) } TokenizerMode::Code => { let started = telemetry.then(std::time::Instant::now); flat_surface::append_cached_code_document( raw, newline, vocabulary, self.config.max_fallback_chars, cache, &mut output.ids, &mut output.lengths, )?; if let Some(started) = started { cache.phase_scan_ns = cache .phase_scan_ns .saturating_add(started.elapsed().as_nanos()); cache.phase_documents = cache.phase_documents.saturating_add(1); } output.document_offsets.push( u64::try_from(output.ids.len()) .map_err(|_| TokenizerError::LengthOverflow("surface token offset"))?, ); Ok(()) } } } #[allow(clippy::too_many_arguments)] fn append_flat_surface_units( &self, raw: &[u8], mut units: Vec, newline: bool, vocabulary: &SurfaceVocabulary, use_morphology: bool, telemetry: bool, cache: &mut FlatAnalysisCache, output: &mut TrainingBatch, ) -> Result<(), TokenizerError> { let mut cuts = Vec::new(); let started = telemetry.then(std::time::Instant::now); let surface_programs = flat_surface::apply_contextual_analysis( self, raw, &mut units, &mut cuts, vocabulary, use_morphology, self.config.max_fallback_chars, cache, )?; if let Some(started) = started { cache.phase_analysis_ns = cache .phase_analysis_ns .saturating_add(started.elapsed().as_nanos()); } let started = telemetry.then(std::time::Instant::now); vocabulary.encode_flat_units_with_programs( raw, &units, &cuts, &surface_programs, self.config.max_fallback_chars, newline, use_morphology, &mut output.ids, &mut output.lengths, )?; if let Some(started) = started { cache.phase_vocab_ns = cache .phase_vocab_ns .saturating_add(started.elapsed().as_nanos()); cache.phase_documents = cache.phase_documents.saturating_add(1); } output.document_offsets.push( u64::try_from(output.ids.len()) .map_err(|_| TokenizerError::LengthOverflow("surface token offset"))?, ); Ok(()) } fn apply_contextual_analysis( &self, raw: &[u8], units: &mut [TokenizedUnit], cache: &mut AnalysisCache, ) -> Result<(), TokenizerError> { let mut segment = Vec::new(); for index in 0..units.len() { let unit = &units[index]; if unit.mode != TokenMode::Turkish || matches!( unit.kind, LexicalKind::LineBreak | LexicalKind::Control | LexicalKind::Opaque ) { self.flush_segment(raw, units, &mut segment, cache)?; continue; } if matches!(unit.kind, LexicalKind::Whitespace) { continue; } segment.push(index); let boundary = is_sentence_boundary(unit.kind, raw, unit.span)?; if boundary || segment.len() >= self.config.max_sentence_tokens { self.flush_segment(raw, units, &mut segment, cache)?; } } self.flush_segment(raw, units, &mut segment, cache) } fn flush_segment( &self, raw: &[u8], units: &mut [TokenizedUnit], indices: &mut Vec, cache: &mut AnalysisCache, ) -> Result<(), TokenizerError> { if indices.is_empty() { return Ok(()); } let tokens = indices .iter() .map(|index| unit_str(raw, units[*index].span)) .collect::, _>>()?; let mut candidate_sets = Vec::with_capacity(tokens.len()); for token in &tokens { candidate_sets.push(cache.analyze(&self.morphology, token)?); } if self.config.contextual_disambiguation { let ambiguity = candidate_sets .iter() .map(|set| set.ambiguity.as_ref()) .collect::>(); let selected = self.disambiguator.disambiguate_indices_causal(&ambiguity)?; if selected.len() != indices.len() { return Err(TokenizerError::ContextLengthMismatch); } for ((&index, set), candidate_index) in indices.iter().zip(&candidate_sets).zip(selected) { let analysis = set.analyses.get(candidate_index).ok_or( TokenizerError::InvalidTrainingEncoding( "rich disambiguator selected an out-of-range candidate", ), )?; apply_selected_analysis(raw, &mut units[index], analysis.clone())?; } } else { for (&index, set) in indices.iter().zip(candidate_sets) { let mut applied = false; for analysis in &set.analyses { match apply_selected_analysis(raw, &mut units[index], analysis.clone()) { Ok(()) => { applied = true; break; } Err(TokenizerError::AlignmentMismatch { .. }) => {} Err(error) => return Err(error), } } if !applied { units[index].status = TokenStatus::Unknown; units[index].cuts.clear(); units[index].analysis = Some(unknown_chunk_analysis(raw, units[index].span)?); } } } indices.clear(); Ok(()) } fn apply_flat_contextual_analysis( &self, raw: &[u8], units: &mut [TokenizedUnit], cache: &mut FlatAnalysisCache, ) -> Result<(), TokenizerError> { let mut segment = Vec::new(); for index in 0..units.len() { let unit = &units[index]; if unit.mode != TokenMode::Turkish || matches!( unit.kind, LexicalKind::LineBreak | LexicalKind::Control | LexicalKind::Opaque ) { self.flush_flat_segment(raw, units, &mut segment, cache)?; continue; } if matches!(unit.kind, LexicalKind::Whitespace) { continue; } segment.push(index); let boundary = is_sentence_boundary(unit.kind, raw, unit.span)?; if boundary || segment.len() >= self.config.max_sentence_tokens { self.flush_flat_segment(raw, units, &mut segment, cache)?; } } self.flush_flat_segment(raw, units, &mut segment, cache) } fn flush_flat_segment( &self, raw: &[u8], units: &mut [TokenizedUnit], indices: &mut Vec, cache: &mut FlatAnalysisCache, ) -> Result<(), TokenizerError> { if indices.is_empty() { return Ok(()); } if let Some(program) = cache.segment_program(raw, units, indices)? { apply_flat_segment_program(units, indices, &program)?; indices.clear(); return Ok(()); } #[cfg(feature = "compiled-surface-table")] if let Some(table) = self.compiled_surface_analysis_table.as_ref() { let mut sources = Vec::with_capacity(indices.len()); for index in indices.iter().copied() { let token = unit_str(raw, units[index].span)?; if let Some(set) = table.get(token) { sources.push(FlatAnalysisSource::Compiled(set)); } else { sources.push(FlatAnalysisSource::Live( cache.analyze(&self.morphology, token)?, )); } } let sets = sources .iter() .map(FlatAnalysisSource::set) .collect::>(); self.apply_flat_sets(raw, units, indices, &sets, cache)?; } else { let mut owned_sets = Vec::with_capacity(indices.len()); for index in indices.iter().copied() { let token = unit_str(raw, units[index].span)?; owned_sets.push(cache.analyze(&self.morphology, token)?); } let sets = owned_sets.iter().map(AsRef::as_ref).collect::>(); self.apply_flat_sets(raw, units, indices, &sets, cache)?; } #[cfg(not(feature = "compiled-surface-table"))] { let mut owned_sets = Vec::with_capacity(indices.len()); for index in indices.iter().copied() { let token = unit_str(raw, units[index].span)?; owned_sets.push(cache.analyze(&self.morphology, token)?); } let sets = owned_sets.iter().map(AsRef::as_ref).collect::>(); self.apply_flat_sets(raw, units, indices, &sets, cache)?; } cache.insert_segment_program(capture_flat_segment_program(units, indices)?); indices.clear(); Ok(()) } fn apply_flat_sets( &self, raw: &[u8], units: &mut [TokenizedUnit], indices: &[usize], sets: &[&FlatAnalysisSet], cache: &mut FlatAnalysisCache, ) -> Result<(), TokenizerError> { let output_invariant = sets.iter().all(|set| set.output_invariant); if self.config.contextual_disambiguation && !output_invariant { let ambiguity = sets .iter() .map(|set| set.ambiguity.as_ref()) .collect::>(); let scoring_codes = sets .iter() .map(|set| set.scoring_codes.as_ref()) .collect::>(); let selected = self.disambiguator.disambiguate_indices_scored_causal( &ambiguity, &scoring_codes, &mut cache.disambiguation_scores, )?; if selected.len() != indices.len() { return Err(TokenizerError::ContextLengthMismatch); } for ((&index, set), candidate_index) in indices.iter().zip(sets).zip(selected) { apply_flat_candidate(raw, &mut units[index], set, candidate_index)?; } } else { for (&index, set) in indices.iter().zip(sets) { apply_flat_candidate(raw, &mut units[index], set, 0)?; } } Ok(()) } } fn capture_flat_segment_program( units: &[TokenizedUnit], indices: &[usize], ) -> Result { let mut tokens = Vec::with_capacity(indices.len()); let mut relative_cuts = Vec::new(); for index in indices { let unit = units .get(*index) .ok_or(TokenizerError::InvalidTrainingEncoding( "segment program capture index is out of range", ))?; let cut_start = u32::try_from(relative_cuts.len()) .map_err(|_| TokenizerError::LengthOverflow("segment program cut offset"))?; for cut in &unit.cuts { let relative = cut.checked_sub(unit.span.start) .ok_or(TokenizerError::InvalidTrainingEncoding( "segment program cut precedes its unit", ))?; relative_cuts.push( u32::try_from(relative) .map_err(|_| TokenizerError::LengthOverflow("segment program relative cut"))?, ); } tokens.push(FlatSegmentTokenProgram { cut_start, cut_len: u16::try_from(unit.cuts.len()) .map_err(|_| TokenizerError::LengthOverflow("segment program cut count"))?, unknown: unit.status == TokenStatus::Unknown, }); } Ok(FlatSegmentProgram { tokens: tokens.into_boxed_slice(), relative_cuts: relative_cuts.into_boxed_slice(), surface_ids: Box::new([]), surface_lengths: Box::new([]), }) } fn apply_flat_segment_program( units: &mut [TokenizedUnit], indices: &[usize], program: &FlatSegmentProgram, ) -> Result<(), TokenizerError> { if indices.len() != program.tokens.len() { return Err(TokenizerError::ContextLengthMismatch); } for (index, token) in indices.iter().copied().zip(program.tokens.iter()) { let unit = units .get_mut(index) .ok_or(TokenizerError::InvalidTrainingEncoding( "segment program apply index is out of range", ))?; let start = usize::try_from(token.cut_start) .map_err(|_| TokenizerError::LengthOverflow("segment program cut start"))?; let end = start .checked_add(usize::from(token.cut_len)) .ok_or(TokenizerError::LengthOverflow("segment program cut end"))?; let relative = program.relative_cuts.get(start..end).ok_or( TokenizerError::InvalidTrainingEncoding("segment program cut range is invalid"), )?; unit.status = if token.unknown { TokenStatus::Unknown } else { TokenStatus::Morphological }; unit.cuts.clear(); unit.cuts.reserve(relative.len()); for cut in relative { unit.cuts .push(unit.span.start.checked_add(u64::from(*cut)).ok_or( TokenizerError::LengthOverflow("segment program absolute cut"), )?); } } Ok(()) } fn apply_flat_candidate( raw: &[u8], unit: &mut TokenizedUnit, set: &FlatAnalysisSet, candidate_index: usize, ) -> Result<(), TokenizerError> { let relative = set.relative_cuts .get(candidate_index) .ok_or(TokenizerError::InvalidTrainingEncoding( "flat disambiguator selected an out-of-range candidate", ))?; let unknown = *set.unknown .get(candidate_index) .ok_or(TokenizerError::InvalidTrainingEncoding( "flat candidate status is out of range", ))?; apply_flat_output(raw, unit, relative, unknown) } fn apply_flat_output( raw: &[u8], unit: &mut TokenizedUnit, relative: &[u32], unknown: bool, ) -> Result<(), TokenizerError> { if matches!(unit.kind, LexicalKind::Punctuation | LexicalKind::Symbol) { unit.status = TokenStatus::Structural; unit.cuts.clear(); return Ok(()); } unit.status = if unknown { TokenStatus::Unknown } else { TokenStatus::Morphological }; unit.cuts.clear(); unit.cuts.reserve(relative.len()); for cut in relative { unit.cuts.push( unit.span .start .checked_add(u64::from(*cut)) .ok_or(TokenizerError::LengthOverflow("flat absolute cut"))?, ); } if unit.kind == LexicalKind::Number { unit.cuts.extend(numeric_micro_cuts(raw, unit.span)?); unit.cuts.sort_unstable(); unit.cuts.dedup(); } Ok(()) } fn encode_flat_surface_range( tokenizer: &Tokenizer<'_>, inputs: &[Vec], newline_flags: &[bool], range: Range, vocabulary: &SurfaceVocabulary, use_morphology: bool, cache: &mut FlatAnalysisCache, ) -> Result { let document_count = range.len(); let token_capacity = range.clone().try_fold(0_usize, |total, index| { total .checked_add(inputs[index].len().saturating_add(2)) .ok_or(TokenizerError::LengthOverflow("surface batch capacity")) })?; let mut batch = TrainingBatch { ids: Vec::with_capacity(token_capacity), lengths: Vec::with_capacity(token_capacity), document_offsets: Vec::with_capacity(document_count.saturating_add(1)), }; batch.document_offsets.push(0); for index in range { tokenizer.append_flat_surface_document( &inputs[index], newline_flags[index], vocabulary, use_morphology, cache, &mut batch, )?; } Ok(batch) } fn concatenate_surface_chunks( mut chunks: Vec<(usize, TrainingBatch)>, ) -> Result { chunks.sort_unstable_by_key(|entry| entry.0); let document_count = chunks.iter().try_fold(0_usize, |total, (_, batch)| { total .checked_add(batch.document_offsets.len().saturating_sub(1)) .ok_or(TokenizerError::LengthOverflow("surface document count")) })?; let token_count = chunks.iter().try_fold(0_usize, |total, (_, batch)| { total .checked_add(batch.ids.len()) .ok_or(TokenizerError::LengthOverflow("surface token count")) })?; let mut output = TrainingBatch { ids: Vec::with_capacity(token_count), lengths: Vec::with_capacity(token_count), document_offsets: Vec::with_capacity(document_count.saturating_add(1)), }; output.document_offsets.push(0); for (_, mut batch) in chunks { if batch.ids.len() != batch.lengths.len() || batch.document_offsets.first() != Some(&0) { return Err(TokenizerError::InvalidTrainingEncoding( "surface chunk metadata is inconsistent", )); } let base = u64::try_from(output.ids.len()) .map_err(|_| TokenizerError::LengthOverflow("surface chunk base"))?; for offset in batch.document_offsets.into_iter().skip(1) { output.document_offsets.push( base.checked_add(offset) .ok_or(TokenizerError::LengthOverflow("surface chunk offset"))?, ); } output.ids.append(&mut batch.ids); output.lengths.append(&mut batch.lengths); } Ok(output) } fn concatenate_training_chunks( mut chunks: Vec<(usize, Vec)>, ) -> Result { chunks.sort_unstable_by_key(|entry| entry.0); let document_count = chunks.iter().try_fold(0_usize, |total, (_, documents)| { total .checked_add(documents.len()) .ok_or(TokenizerError::LengthOverflow("flat document count")) })?; let token_count = chunks.iter().try_fold(0_usize, |total, (_, documents)| { documents.iter().try_fold(total, |subtotal, document| { subtotal .checked_add(document.ids.len()) .ok_or(TokenizerError::LengthOverflow("flat token count")) }) })?; let mut batch = TrainingBatch { ids: Vec::with_capacity(token_count), lengths: Vec::with_capacity(token_count), document_offsets: Vec::with_capacity( document_count .checked_add(1) .ok_or(TokenizerError::LengthOverflow("flat offset count"))?, ), }; batch.document_offsets.push(0); for (_, documents) in chunks { for mut document in documents { if document.ids.len() != document.lengths.len() { return Err(TokenizerError::InvalidTrainingEncoding( "flat document ID and length counts differ", )); } batch.ids.append(&mut document.ids); batch.lengths.append(&mut document.lengths); batch.document_offsets.push( u64::try_from(batch.ids.len()) .map_err(|_| TokenizerError::LengthOverflow("flat token offset"))?, ); } } Ok(batch) } fn build_batch_ranges( inputs: &[Vec], threads: usize, ) -> Result>, TokenizerError> { let total_bytes = inputs.iter().try_fold(0_usize, |total, input| { total .checked_add(input.len()) .ok_or(TokenizerError::LengthOverflow("batch bytes")) })?; let desired_chunks = threads.saturating_mul(BATCH_CHUNKS_PER_WORKER).max(1); let target_bytes = total_bytes .div_ceil(desired_chunks) .max(MIN_BATCH_CHUNK_BYTES); let mut ranges = Vec::new(); let mut start = 0_usize; let mut bytes = 0_usize; for (index, input) in inputs.iter().enumerate() { bytes = bytes .checked_add(input.len()) .ok_or(TokenizerError::LengthOverflow("batch chunk bytes"))?; let end = index + 1; if bytes >= target_bytes && end < inputs.len() { ranges.push(start..end); start = end; bytes = 0; } } ranges.push(start..inputs.len()); Ok(ranges) } fn flat_candidates_output_invariant(relative_cuts: &[Vec], unknown: &[bool]) -> bool { let Some(first_cuts) = relative_cuts.first() else { return false; }; let Some(&first_unknown) = unknown.first() else { return false; }; relative_cuts .iter() .zip(unknown) .all(|(cuts, candidate_unknown)| cuts == first_cuts && *candidate_unknown == first_unknown) } fn build_flat_analysis_set( token: &str, analyses: Vec, ) -> Result { let numeric = token.chars().any(char::is_numeric); let mut relative_cuts = Vec::new(); let mut ambiguity = Vec::new(); let mut unknown = Vec::new(); for analysis in analyses { let alignment = if numeric { let mut aligned_analysis = analysis.clone(); complete_numeric_derivation(token, &mut aligned_analysis); align_analysis(token, 0, &aligned_analysis) } else { align_analysis(token, 0, &analysis) }; let aligned = match alignment { Ok(value) => value, Err(TokenizerError::AlignmentMismatch { .. }) => continue, Err(error) => return Err(error), }; let cuts = aligned .cuts .into_iter() .map(|cut| { u32::try_from(cut).map_err(|_| TokenizerError::LengthOverflow("flat relative cut")) }) .collect::, _>>()?; relative_cuts.push(cuts); ambiguity.push(ambiguity_word_data(&analysis)); unknown.push(analysis.dictionary_id == "UNK_Unk_Unk"); } if relative_cuts.is_empty() { let unknown_candidate = unknown_native_analysis(token); relative_cuts.push(Vec::new()); ambiguity.push(ambiguity_word_data(&unknown_candidate)); unknown.push(true); } if relative_cuts.len() != ambiguity.len() || ambiguity.len() != unknown.len() { return Err(TokenizerError::InvalidTrainingEncoding( "flat candidate metadata cardinalities differ", )); } let output_invariant = flat_candidates_output_invariant(&relative_cuts, &unknown); Ok(FlatAnalysisSet { relative_cuts: relative_cuts.into_boxed_slice(), ambiguity: ambiguity.into_boxed_slice(), scoring_codes: Box::new([]), unknown: unknown.into_boxed_slice(), output_invariant, }) } fn surface_valid_candidates( token: &str, absolute_start: u64, analyses: Vec, ) -> Vec { let mut valid = analyses .into_iter() .filter(|analysis| { if analysis.primary_pos == "Num" { let mut candidate = analysis.clone(); complete_numeric_derivation(token, &mut candidate); align_analysis(token, absolute_start, &candidate).is_ok() } else { align_analysis(token, absolute_start, analysis).is_ok() } }) .collect::>(); if valid.is_empty() { valid.push(unknown_native_analysis(token)); } valid } fn unknown_native_analysis(token: &str) -> NativeAnalysis { NativeAnalysis { canonical: format!("UNK_Unk_Unk\u{1}Unknown={token}\u{2}"), dictionary_id: "UNK_Unk_Unk".to_owned(), lemma: token.to_owned(), primary_pos: "Unk".to_owned(), secondary_pos: "Unk".to_owned(), surface_form: token.to_owned(), stem: token.to_owned(), ending: String::new(), morphemes: vec![NativeMorpheme { id: "Unknown".to_owned(), name: "Unknown".to_owned(), surface: token.to_owned(), derivational: false, informal: false, pos: None, mapped_id: None, }], } } fn apply_selected_analysis( raw: &[u8], unit: &mut TokenizedUnit, analysis: NativeAnalysis, ) -> Result<(), TokenizerError> { apply_analysis(raw, unit, analysis)?; if matches!(unit.kind, LexicalKind::Punctuation | LexicalKind::Symbol) { unit.status = TokenStatus::Structural; unit.cuts.clear(); unit.analysis = None; return Ok(()); } if unit.kind == LexicalKind::Number { let mut cuts = numeric_micro_cuts(raw, unit.span)?; cuts.extend_from_slice(&unit.cuts); cuts.sort_unstable(); cuts.dedup(); unit.cuts = cuts; } Ok(()) } fn split_compact_units( lexical_scan: &CompactScanResult, code_spans: &[ByteSpan], requested: TokenizerMode, ) -> Result, TokenizerError> { let spans = lexical_scan.spans(); let kinds = lexical_scan.kinds(); if spans.len() != kinds.len() { return Err(TokenizerError::Scan(ScanError::MetadataLengthMismatch { unit_count: spans.len(), kind_count: kinds.len(), })); } if code_spans.is_empty() { return split_compact_units_without_code_spans(lexical_scan, requested); } let mut boundaries = Vec::new(); for (index, (span, kind)) in spans.iter().copied().zip(kinds.iter().copied()).enumerate() { boundaries.push(span.start); boundaries.push(span.end); let bytes = lexical_scan.unit_bytes(index)?; if kind == LexicalKind::Punctuation { add_scalar_boundaries(bytes, span.start, &mut boundaries)?; } let mode = effective_mode(span, kind, code_spans, requested); if mode == TokenMode::Turkish && kind == LexicalKind::Symbol && bytes.iter().all(u8::is_ascii_punctuation) { add_scalar_boundaries(bytes, span.start, &mut boundaries)?; } if mode == TokenMode::Code && matches!(kind, LexicalKind::Word | LexicalKind::Number) { let text = std::str::from_utf8(bytes).map_err(|_| TokenizerError::InvalidUtf8Unit)?; for relative in identifier_cuts(text) { boundaries.push( span.start .checked_add( u64::try_from(relative).map_err(|_| { TokenizerError::LengthOverflow("identifier boundary") })?, ) .ok_or(TokenizerError::LengthOverflow("identifier boundary"))?, ); } } } for span in code_spans { boundaries.push(span.start); boundaries.push(span.end); } boundaries.sort_unstable(); boundaries.dedup(); let mut units = Vec::with_capacity(boundaries.len().saturating_sub(1)); let mut source_index = 0_usize; for window in boundaries.windows(2) { let span = ByteSpan { start: window[0], end: window[1], }; while source_index + 1 < spans.len() && spans[source_index].end <= span.start { source_index += 1; } let source = spans .get(source_index) .copied() .ok_or(TokenizerError::InvalidTrainingEncoding( "compact split source unit is absent", ))?; if span.start < source.start || span.end > source.end { return Err(TokenizerError::InvalidTrainingEncoding( "compact split span crosses a lexical unit", )); } let kind = kinds[source_index]; let mode = effective_mode(span, kind, code_spans, requested); let status = match mode { TokenMode::Code => TokenStatus::Code, TokenMode::Opaque => TokenStatus::Opaque, TokenMode::Turkish => TokenStatus::Structural, }; units.push(TokenizedUnit { span, kind, mode, status, group_id: None, cuts: Vec::new(), analysis: None, }); } Ok(units) } fn split_compact_units_without_code_spans( lexical_scan: &CompactScanResult, requested: TokenizerMode, ) -> Result, TokenizerError> { let spans = lexical_scan.spans(); let kinds = lexical_scan.kinds(); let mut units = Vec::with_capacity(spans.len()); let mut boundaries = Vec::with_capacity(8); for (index, (span, kind)) in spans.iter().copied().zip(kinds.iter().copied()).enumerate() { let mode = effective_mode(span, kind, &[], requested); boundaries.clear(); boundaries.push(span.start); boundaries.push(span.end); let bytes = lexical_scan.unit_bytes(index)?; if kind == LexicalKind::Punctuation { add_scalar_boundaries(bytes, span.start, &mut boundaries)?; } else if mode == TokenMode::Turkish && kind == LexicalKind::Symbol && bytes.iter().all(u8::is_ascii_punctuation) { add_scalar_boundaries(bytes, span.start, &mut boundaries)?; } if mode == TokenMode::Code && matches!(kind, LexicalKind::Word | LexicalKind::Number) { let text = std::str::from_utf8(bytes).map_err(|_| TokenizerError::InvalidUtf8Unit)?; for relative in identifier_cuts(text) { boundaries.push( span.start .checked_add( u64::try_from(relative).map_err(|_| { TokenizerError::LengthOverflow("identifier boundary") })?, ) .ok_or(TokenizerError::LengthOverflow("identifier boundary"))?, ); } } if boundaries.len() > 2 { boundaries.sort_unstable(); boundaries.dedup(); } let status = match mode { TokenMode::Code => TokenStatus::Code, TokenMode::Opaque => TokenStatus::Opaque, TokenMode::Turkish => TokenStatus::Structural, }; for window in boundaries.windows(2) { units.push(TokenizedUnit { span: ByteSpan { start: window[0], end: window[1], }, kind, mode, status, group_id: None, cuts: Vec::new(), analysis: None, }); } } Ok(units) } fn split_units( lexical_scan: &nedo_core::ScanResult, code_spans: &[ByteSpan], requested: TokenizerMode, ) -> Result, TokenizerError> { if code_spans.is_empty() { return split_units_without_code_spans(lexical_scan, requested); } let mut boundaries = Vec::new(); for (index, unit) in lexical_scan.document().units().iter().enumerate() { boundaries.push(unit.span.start); boundaries.push(unit.span.end); let kind = lexical_scan.kind(index)?; let mode = effective_mode(unit.span, kind, code_spans, requested); if kind == LexicalKind::Punctuation { add_scalar_boundaries( lexical_scan.unit_bytes(index)?, unit.span.start, &mut boundaries, )?; } else if mode == TokenMode::Turkish && kind == LexicalKind::Symbol { let bytes = lexical_scan.unit_bytes(index)?; if bytes.iter().all(u8::is_ascii_punctuation) { add_scalar_boundaries(bytes, unit.span.start, &mut boundaries)?; } } if mode == TokenMode::Code && matches!(kind, LexicalKind::Word | LexicalKind::Number) { let text = std::str::from_utf8(lexical_scan.unit_bytes(index)?) .map_err(|_| TokenizerError::InvalidUtf8Unit)?; for relative in identifier_cuts(text) { boundaries.push( unit.span.start + u64::try_from(relative) .map_err(|_| TokenizerError::LengthOverflow("identifier boundary"))?, ); } } } for span in code_spans { boundaries.push(span.start); boundaries.push(span.end); } boundaries.sort_unstable(); boundaries.dedup(); let mut units = Vec::new(); let mut source_index = 0_usize; for window in boundaries.windows(2) { let unit_span = ByteSpan { start: window[0], end: window[1], }; while lexical_scan.document().units()[source_index].span.end <= unit_span.start { source_index += 1; } let kind = lexical_scan.kind(source_index)?; let mode = effective_mode(unit_span, kind, code_spans, requested); let status = match mode { TokenMode::Code => TokenStatus::Code, TokenMode::Opaque => TokenStatus::Opaque, TokenMode::Turkish => TokenStatus::Structural, }; units.push(TokenizedUnit { span: unit_span, kind, mode, status, group_id: None, cuts: Vec::new(), analysis: None, }); } Ok(units) } fn split_units_without_code_spans( lexical_scan: &nedo_core::ScanResult, requested: TokenizerMode, ) -> Result, TokenizerError> { let source_units = lexical_scan.document().units(); let kinds = lexical_scan.kinds(); if source_units.len() != kinds.len() { return Err(TokenizerError::Scan(ScanError::MetadataLengthMismatch { unit_count: source_units.len(), kind_count: kinds.len(), })); } let mut units = Vec::with_capacity(source_units.len()); let mut boundaries = Vec::with_capacity(8); for (index, (source, kind)) in source_units.iter().zip(kinds.iter().copied()).enumerate() { let mode = effective_mode(source.span, kind, &[], requested); boundaries.clear(); boundaries.push(source.span.start); boundaries.push(source.span.end); let bytes = lexical_scan.unit_bytes(index)?; if kind == LexicalKind::Punctuation { add_scalar_boundaries(bytes, source.span.start, &mut boundaries)?; } else if mode == TokenMode::Turkish && kind == LexicalKind::Symbol && bytes.iter().all(u8::is_ascii_punctuation) { add_scalar_boundaries(bytes, source.span.start, &mut boundaries)?; } if mode == TokenMode::Code && matches!(kind, LexicalKind::Word | LexicalKind::Number) { let text = std::str::from_utf8(bytes).map_err(|_| TokenizerError::InvalidUtf8Unit)?; for relative in identifier_cuts(text) { boundaries.push( source .span .start .checked_add( u64::try_from(relative).map_err(|_| { TokenizerError::LengthOverflow("identifier boundary") })?, ) .ok_or(TokenizerError::LengthOverflow("identifier boundary"))?, ); } } if boundaries.len() > 2 { boundaries.sort_unstable(); boundaries.dedup(); } let status = match mode { TokenMode::Code => TokenStatus::Code, TokenMode::Opaque => TokenStatus::Opaque, TokenMode::Turkish => TokenStatus::Structural, }; for window in boundaries.windows(2) { units.push(TokenizedUnit { span: ByteSpan { start: window[0], end: window[1], }, kind, mode, status, group_id: None, cuts: Vec::new(), analysis: None, }); } } Ok(units) } fn add_scalar_boundaries( bytes: &[u8], absolute_start: u64, boundaries: &mut Vec, ) -> Result<(), TokenizerError> { let text = std::str::from_utf8(bytes).map_err(|_| TokenizerError::InvalidUtf8Unit)?; for (relative, _) in text.char_indices().skip(1) { boundaries.push( absolute_start + u64::try_from(relative) .map_err(|_| TokenizerError::LengthOverflow("structural boundary"))?, ); } Ok(()) } fn effective_mode( span: ByteSpan, kind: LexicalKind, code_spans: &[ByteSpan], requested: TokenizerMode, ) -> TokenMode { if matches!(kind, LexicalKind::Opaque | LexicalKind::Control) { return TokenMode::Opaque; } match requested { TokenizerMode::Code => TokenMode::Code, TokenizerMode::Turkish => TokenMode::Turkish, TokenizerMode::Auto => { if code_spans .iter() .any(|code| span.start >= code.start && span.end <= code.end) { TokenMode::Code } else { TokenMode::Turkish } } } } fn apply_analysis( raw: &[u8], unit: &mut TokenizedUnit, mut analysis: NativeAnalysis, ) -> Result<(), TokenizerError> { let token = unit_str(raw, unit.span)?; if analysis.dictionary_id == "UNK_Unk_Unk" { unit.status = TokenStatus::Unknown; unit.analysis = Some(AnalysisMetadata { canonical: analysis.canonical, dictionary_id: analysis.dictionary_id, lemma: analysis.lemma, primary_pos: analysis.primary_pos, secondary_pos: analysis.secondary_pos, morphemes: vec![AlignedMorpheme { id: "Unknown".to_owned(), surface: token.to_owned(), span: unit.span, derivational: false, }], }); return Ok(()); } complete_numeric_derivation(token, &mut analysis); let aligned = align_analysis(token, unit.span.start, &analysis)?; unit.cuts = aligned.cuts; unit.status = TokenStatus::Morphological; unit.analysis = Some(AnalysisMetadata { canonical: analysis.canonical, dictionary_id: analysis.dictionary_id, lemma: analysis.lemma, primary_pos: analysis.primary_pos, secondary_pos: analysis.secondary_pos, morphemes: aligned.morphemes, }); Ok(()) } fn numeric_micro_cuts(raw: &[u8], span: ByteSpan) -> Result, TokenizerError> { let text = unit_str(raw, span)?; let chars = text.char_indices().collect::>(); let mut cuts = Vec::new(); for (position, &(index, value)) in chars.iter().enumerate() { if !matches!(value, '.' | ',' | ':' | '/' | '-') { continue; } let previous_is_digit = position > 0 && chars[position - 1].1.is_numeric(); let next_is_digit = chars .get(position + 1) .is_some_and(|entry| entry.1.is_numeric()); if previous_is_digit && next_is_digit { let start = span.start + u64::try_from(index) .map_err(|_| TokenizerError::LengthOverflow("numeric separator start"))?; let end = start + u64::try_from(value.len_utf8()) .map_err(|_| TokenizerError::LengthOverflow("numeric separator end"))?; cuts.push(start); cuts.push(end); } } cuts.sort_unstable(); cuts.dedup(); Ok(cuts) } fn split_long_flat_units( raw: &[u8], units: Vec, maximum_chars: usize, ) -> Result, TokenizerError> { let mut requires_split = false; for unit in &units { let should_split = matches!(unit.status, TokenStatus::Unknown | TokenStatus::Code) || unit.mode == TokenMode::Opaque; if !should_split { continue; } let bytes = unit_bytes(raw, unit.span)?; let limit = maximum_chars; if unit.mode == TokenMode::Opaque { requires_split = bytes.len() > limit; } else if bytes.len() > limit { let text = std::str::from_utf8(bytes).map_err(|_| TokenizerError::InvalidUtf8Unit)?; requires_split = text.chars().nth(limit).is_some(); } if requires_split { break; } } if !requires_split { return Ok(units); } let mut output = Vec::with_capacity(units.len()); for unit in units { let should_split = matches!(unit.status, TokenStatus::Unknown | TokenStatus::Code) || unit.mode == TokenMode::Opaque; if !should_split { output.push(unit); continue; } let limit = maximum_chars; let spans = chunk_span(raw, unit.span, limit, unit.mode == TokenMode::Opaque)?; if spans.len() == 1 { output.push(unit); continue; } for span in spans { output.push(TokenizedUnit { span, kind: unit.kind, mode: unit.mode, status: unit.status, group_id: None, cuts: Vec::new(), analysis: None, }); } } Ok(output) } fn split_long_fallback_units( raw: &[u8], units: Vec, maximum_chars: usize, ) -> Result, TokenizerError> { let mut output = Vec::with_capacity(units.len()); for unit in units { let should_split = matches!(unit.status, TokenStatus::Unknown | TokenStatus::Code) || unit.mode == TokenMode::Opaque; if !should_split { output.push(unit); continue; } let limit = maximum_chars; let spans = chunk_span(raw, unit.span, limit, unit.mode == TokenMode::Opaque)?; if spans.len() == 1 { output.push(unit); continue; } for span in spans { let analysis = if unit.status == TokenStatus::Unknown { Some(unknown_chunk_analysis(raw, span)?) } else { None }; output.push(TokenizedUnit { span, kind: unit.kind, mode: unit.mode, status: unit.status, group_id: None, cuts: Vec::new(), analysis, }); } } Ok(output) } fn chunk_span( raw: &[u8], span: ByteSpan, maximum: usize, byte_general: bool, ) -> Result, TokenizerError> { let bytes = unit_bytes(raw, span)?; if byte_general { if bytes.len() <= maximum { return Ok(vec![span]); } let mut result = Vec::new(); let mut start = span.start; for chunk in bytes.chunks(maximum) { let end = start .checked_add( u64::try_from(chunk.len()) .map_err(|_| TokenizerError::LengthOverflow("opaque fallback chunk"))?, ) .ok_or(TokenizerError::LengthOverflow("opaque fallback end"))?; result.push(ByteSpan { start, end }); start = end; } return Ok(result); } let text = std::str::from_utf8(bytes).map_err(|_| TokenizerError::InvalidUtf8Unit)?; if text.chars().count() <= maximum { return Ok(vec![span]); } let mut result = Vec::new(); let mut chunk_start = 0_usize; let mut count = 0_usize; for (index, _) in text.char_indices() { if count == maximum { result.push(ByteSpan { start: span.start + u64::try_from(chunk_start) .map_err(|_| TokenizerError::LengthOverflow("fallback chunk start"))?, end: span.start + u64::try_from(index) .map_err(|_| TokenizerError::LengthOverflow("fallback chunk end"))?, }); chunk_start = index; count = 0; } count += 1; } if chunk_start < text.len() { result.push(ByteSpan { start: span.start + u64::try_from(chunk_start) .map_err(|_| TokenizerError::LengthOverflow("fallback final start"))?, end: span.end, }); } Ok(result) } fn unknown_chunk_analysis(raw: &[u8], span: ByteSpan) -> Result { let surface = unit_str(raw, span)?.to_owned(); Ok(AnalysisMetadata { canonical: format!("UNK_Unk_Unk\u{1}Unknown={surface}\u{2}"), dictionary_id: "UNK_Unk_Unk".to_owned(), lemma: surface.clone(), primary_pos: "Unk".to_owned(), secondary_pos: "Unk".to_owned(), morphemes: vec![AlignedMorpheme { id: "Unknown".to_owned(), surface, span, derivational: false, }], }) } fn complete_numeric_derivation(token: &str, analysis: &mut NativeAnalysis) { if analysis.primary_pos != "Num" { return; } let Some((stem, suffix)) = split_apostrophe_suffix(token) else { return; }; if analysis.stem != stem { return; } let Some(normalized_suffix) = turkish_lowercase_single_scalar(suffix) else { return; }; let Some(derivation) = numeric_derivation_prefix(&normalized_suffix) else { return; }; if analysis .morphemes .iter() .any(|morpheme| morpheme.id == derivation.id) { return; } let derivation_char_count = derivation.surface.chars().count(); let original_prefix_end = suffix .char_indices() .nth(derivation_char_count) .map_or(suffix.len(), |(index, _)| index); let original_remaining = &suffix[original_prefix_end..]; let Some(normalized_remaining) = turkish_lowercase_single_scalar(original_remaining) else { return; }; let Some(normalized_analysis_ending) = turkish_lowercase_single_scalar(&analysis.ending) else { return; }; if normalized_analysis_ending != normalized_remaining { return; } let old_ending = analysis.ending.clone(); analysis.ending.clear(); analysis.ending.push_str(derivation.surface); analysis.ending.push_str(&old_ending); analysis.surface_form.clear(); analysis.surface_form.push_str(stem); analysis.surface_form.push_str(&analysis.ending); analysis.morphemes.insert( 1, NativeMorpheme { id: derivation.id.to_owned(), name: derivation.name.to_owned(), surface: derivation.surface.to_owned(), derivational: true, informal: false, pos: Some(derivation.pos.to_owned()), mapped_id: None, }, ); analysis.canonical = tokenizer_canonical_key(&analysis.dictionary_id, &analysis.morphemes); } #[derive(Clone, Copy)] struct NumericDerivation<'a> { id: &'static str, name: &'static str, pos: &'static str, surface: &'a str, } fn numeric_derivation_prefix(suffix: &str) -> Option> { const ORDINALS: [&str; 8] = ["ıncı", "inci", "uncu", "üncü", "ncı", "nci", "ncu", "ncü"]; const DISTRIBUTIVES: [&str; 4] = ["şar", "şer", "ar", "er"]; if let Some(surface) = ORDINALS .into_iter() .find(|candidate| suffix.starts_with(candidate)) { return Some(NumericDerivation { id: "Ord", name: "Ordinal", pos: "Adj", surface, }); } DISTRIBUTIVES .into_iter() .find(|candidate| suffix.starts_with(candidate)) .map(|surface| NumericDerivation { id: "Dist", name: "Distribution", pos: "Num", surface, }) } fn turkish_lowercase_single_scalar(value: &str) -> Option { let mut output = String::with_capacity(value.len()); for character in value.chars() { let lowered = match character { 'I' => 'ı', 'İ' => 'i', _ => { let mut lowercase = character.to_lowercase(); let first = lowercase.next()?; if lowercase.next().is_some() { return None; } first } }; output.push(lowered); } Some(output) } fn tokenizer_canonical_key(dictionary_id: &str, morphemes: &[NativeMorpheme]) -> String { let mut canonical = String::with_capacity(dictionary_id.len() + morphemes.len() * 8); canonical.push_str(dictionary_id); canonical.push('\u{1}'); for morpheme in morphemes { canonical.push_str(&morpheme.id); canonical.push('='); canonical.push_str(&morpheme.surface); canonical.push('\u{2}'); } canonical } fn split_apostrophe_suffix(token: &str) -> Option<(&str, &str)> { for (index, value) in token.char_indices() { if matches!( value, '\'' | '\u{2018}' | '\u{2019}' | '\u{02bc}' | '\u{ff07}' ) { let end = index + value.len_utf8(); if index > 0 && end < token.len() { return Some((&token[..index], &token[end..])); } return None; } } None } fn assign_inner_groups(raw: &[u8], units: &mut [TokenizedUnit]) -> Result<(), TokenizerError> { let mut next_group = 0_u32; let mut previous_content: Option = None; for index in 0..units.len() { if units[index].mode != TokenMode::Turkish { previous_content = None; continue; } if matches!(units[index].kind, LexicalKind::Whitespace) { continue; } if matches!(units[index].kind, LexicalKind::LineBreak) { previous_content = None; continue; } let token = unit_str(raw, units[index].span)?; if is_phonological_clitic(token) { if let Some(previous) = previous_content { let group = units[previous].group_id.unwrap_or(next_group); if units[previous].group_id.is_none() { next_group = next_group .checked_add(1) .ok_or(TokenizerError::LengthOverflow("group id"))?; units[previous].group_id = Some(group); } for unit in &mut units[previous + 1..=index] { unit.group_id = Some(group); } previous_content = Some(index); continue; } } units[index].group_id = Some(next_group); next_group = next_group .checked_add(1) .ok_or(TokenizerError::LengthOverflow("group id"))?; previous_content = Some(index); } Ok(()) } fn is_phonological_clitic(token: &str) -> bool { matches!( token.to_lowercase().as_str(), "mi" | "mı" | "mu" | "mü" | "de" | "da" ) } fn is_sentence_boundary( kind: LexicalKind, raw: &[u8], span: ByteSpan, ) -> Result { if kind != LexicalKind::Punctuation { return Ok(false); } let bytes = unit_bytes(raw, span)?; Ok(matches!(bytes, b"." | b"!" | b"?")) } fn unit_str(raw: &[u8], span: ByteSpan) -> Result<&str, TokenizerError> { std::str::from_utf8(unit_bytes(raw, span)?).map_err(|_| TokenizerError::InvalidUtf8Unit) } fn unit_bytes(raw: &[u8], span: ByteSpan) -> Result<&[u8], TokenizerError> { let start = usize::try_from(span.start).map_err(|_| TokenizerError::LengthOverflow("unit start"))?; let end = usize::try_from(span.end).map_err(|_| TokenizerError::LengthOverflow("unit end"))?; raw.get(start..end) .ok_or(TokenizerError::UnitOutsideDocument) } fn verify_sha( bytes: &[u8], expected: &'static str, asset: &'static str, ) -> Result<(), TokenizerError> { let actual = format!("{:x}", Sha256::digest(bytes)); if actual != expected { return Err(TokenizerError::AssetChecksumMismatch { asset, expected, actual, }); } Ok(()) } #[cfg(feature = "compiled-surface-table")] fn hex_bytes(bytes: &[u8]) -> String { let mut output = String::with_capacity(bytes.len().saturating_mul(2)); for byte in bytes { use std::fmt::Write as _; let _ = write!(output, "{byte:02x}"); } output } /// Tokenizer failure; every failure is explicit and inspectable. #[derive(Debug)] pub enum TokenizerError { /// Low-level scanner failure. Scan(ScanError), /// Lossless format failure. Format(FormatError), /// Native morphology binary/runtime failure. Morphology(BinaryError), /// Native contextual model failure. Disambiguation(DisambiguationError), /// Embedded or supplied asset checksum mismatch. AssetChecksumMismatch { asset: &'static str, expected: &'static str, actual: String, }, /// Tokenizer codec asset identity differs. AssetIdentityMismatch, /// Morphological surface could not be mapped to original bytes exactly. AlignmentMismatch { token: String, expected: String, actual: String, canonical: String, }, /// A lowercase operation expanded into multiple scalars and is unsupported in schema v1. UnsupportedNormalization { value: char }, /// Length arithmetic or representation overflow. LengthOverflow(&'static str), /// Invalid configuration. InvalidConfiguration(&'static str), /// Worker thread panicked. WorkerPanicked, /// Unit did not cover the required interval. InvalidUnitCoverage { index: usize, expected: u64, start: u64, end: u64, }, /// Units did not cover the full document. IncompleteDocument { covered: u64, document_len: u64 }, /// Analysis-required status lacks metadata. MissingAnalysis { index: usize }, /// Structural/code/opaque status unexpectedly carries morphology. UnexpectedAnalysis { index: usize }, /// Morpheme span escapes its unit. MorphemeOutsideUnit { index: usize }, /// Unit mode/status/analysis metadata are semantically inconsistent. InvalidUnitMetadata { index: usize, reason: &'static str }, /// Unit span cannot be sliced. UnitOutsideDocument, /// Turkish-mode unit was not valid UTF-8. InvalidUtf8Unit, /// Decoder output length differed from contextual input. ContextLengthMismatch, /// Tokenized codec magic is invalid. BadCodecMagic, /// Tokenized codec schema is unsupported. UnsupportedCodecVersion(u32), /// Tokenized codec was truncated. TruncatedCodec, /// Tokenized codec payload checksum failed. CodecChecksumMismatch, /// Tokenized codec count cannot fit in remaining bytes. ImpossibleCodecCount(&'static str), /// Tokenized codec has trailing bytes. TrailingCodecBytes(usize), /// Tokenized codec contains an invalid enum value. InvalidCodecEnum(&'static str, u8), /// Tokenized codec contains a non-canonical boolean byte. InvalidCodecBoolean { field: &'static str, value: u8 }, /// Tokenized codec contains invalid UTF-8 metadata. InvalidCodecUtf8, /// Vocabulary identity/content failure. InvalidVocabulary(&'static str), /// Rich-to-training conversion violated a flat-stream invariant. InvalidTrainingEncoding(&'static str), /// A compiled table did not match an independently trusted payload-digest seal. CompiledSurfaceTableSealMismatch { expected: String, actual: String }, /// A compiled invariant surface entry failed exact startup verification. InvalidCompiledSurfaceTable { surface: String, reason: &'static str, }, } impl fmt::Display for TokenizerError { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { write!(formatter, "{self:?}") } } impl std::error::Error for TokenizerError {} impl From for TokenizerError { fn from(value: ScanError) -> Self { Self::Scan(value) } } impl From for TokenizerError { fn from(value: FormatError) -> Self { Self::Format(value) } } impl From for TokenizerError { fn from(value: BinaryError) -> Self { Self::Morphology(value) } } impl From for TokenizerError { fn from(value: DisambiguationError) -> Self { Self::Disambiguation(value) } } #[cfg(test)] mod tests { use super::{ decode_tokenized, encode_tokenized, CharacterVocabulary, SurfaceEncoderOptions, SurfaceVocabulary, TokenMode, TokenStatus, Tokenizer, TokenizerConfig, TokenizerMode, TrainingEncodingOptions, }; #[test] fn embedded_assets_tokenize_and_round_trip() -> Result<(), super::TokenizerError> { let tokenizer = Tokenizer::embedded(TokenizerConfig::default())?; let raw = "Ankara'da evlerimizden mi?\r\n```rust\nfn main() {}\n``` 😀" .as_bytes() .to_vec(); let document = tokenizer.tokenize(raw.clone())?; assert_eq!(document.decode(), raw); assert!(document .units() .iter() .any(|unit| unit.status == TokenStatus::Morphological)); assert!(document .units() .iter() .any(|unit| unit.mode == TokenMode::Code)); let encoded = encode_tokenized(&document)?; assert_eq!(decode_tokenized(&encoded)?, document); Ok(()) } #[test] fn clock_suffix_is_morphological_in_full_pipeline() -> Result<(), super::TokenizerError> { let tokenizer = Tokenizer::embedded(TokenizerConfig::default())?; let token = "14:30:05'te"; let analyses = super::analyze_token_with_quality_fallback(&tokenizer.morphology, token)?; assert_eq!(analyses.len(), 1); assert_ne!(analyses[0].dictionary_id, "UNK_Unk_Unk"); let set = super::build_flat_analysis_set(token, analyses)?; assert_eq!(set.unknown.as_ref(), &[false]); let document = tokenizer.tokenize(token.as_bytes().to_vec())?; assert_eq!(document.units().len(), 1); assert_eq!(document.units()[0].status, TokenStatus::Morphological); assert_eq!(document.decode(), token.as_bytes()); Ok(()) } #[test] fn embedded_informal_morphology_produces_surface_aligned_cuts( ) -> Result<(), super::TokenizerError> { let tokenizer = Tokenizer::embedded(TokenizerConfig::default())?; for (surface, expected) in [ ("gidicem", ["gid", "ice", "m"]), ("geliyom", ["gel", "iyo", "m"]), ("yapıyon", ["yap", "ıyo", "n"]), ("yapıyosun", ["yap", "ıyo", "sun"]), ("yazacam", ["yaz", "aca", "m"]), ("yazıcam", ["yaz", "ıca", "m"]), ] { let document = tokenizer.tokenize(surface.as_bytes().to_vec())?; assert_eq!(document.decode(), surface.as_bytes()); assert_eq!(document.units().len(), 1); let unit = &document.units()[0]; assert_eq!(unit.status, TokenStatus::Morphological); let mut boundaries = Vec::with_capacity(unit.cuts.len() + 2); boundaries.push(unit.span.start); boundaries.extend(unit.cuts.iter().copied()); boundaries.push(unit.span.end); let pieces = boundaries .windows(2) .map(|pair| { let start = usize::try_from(pair[0]).map_err(|_| { super::TokenizerError::LengthOverflow("informal test start") })?; let end = usize::try_from(pair[1]) .map_err(|_| super::TokenizerError::LengthOverflow("informal test end"))?; std::str::from_utf8(&document.raw()[start..end]) .map_err(|_| super::TokenizerError::InvalidUtf8Unit) }) .collect::, _>>()?; assert_eq!(pieces.as_slice(), expected.as_slice()); let analysis = unit .analysis .as_ref() .ok_or(super::TokenizerError::MissingAnalysis { index: 0 })?; assert!(analysis.canonical.contains("_Informal")); } Ok(()) } #[test] fn technical_borrowings_remain_byte_exact_without_lexical_exceptions() -> Result<(), super::TokenizerError> { let tokenizer = Tokenizer::embedded(TokenizerConfig::default())?; let raw = "kanka deploy ettim ama loglar acayip".as_bytes().to_vec(); let document = tokenizer.tokenize(raw.clone())?; assert_eq!(document.decode(), raw); Ok(()) } #[test] fn auto_mode_detects_unmarked_code_without_reclassifying_turkish( ) -> Result<(), super::TokenizerError> { let tokenizer = Tokenizer::embedded(TokenizerConfig::default())?; let raw = "geliyom gidiyom.\nfoo_bar += 42;\nsonra dönerim." .as_bytes() .to_vec(); let document = tokenizer.tokenize(raw.clone())?; assert_eq!(document.decode(), raw); let mut saw_code = false; let mut saw_informal_turkish = false; for unit in document.units() { let start = usize::try_from(unit.span.start) .map_err(|_| super::TokenizerError::LengthOverflow("auto-code test start"))?; let end = usize::try_from(unit.span.end) .map_err(|_| super::TokenizerError::LengthOverflow("auto-code test end"))?; let surface = std::str::from_utf8(&document.raw()[start..end]) .map_err(|_| super::TokenizerError::InvalidUtf8Unit)?; if surface == "foo" || surface == "_" || surface == "bar" || surface == "+=" || surface == "42" { assert_eq!(unit.mode, TokenMode::Code); assert_eq!(unit.status, TokenStatus::Code); saw_code = true; } if surface == "geliyom" || surface == "gidiyom" { assert_eq!(unit.mode, TokenMode::Turkish); assert_eq!(unit.status, TokenStatus::Morphological); saw_informal_turkish = true; } } assert!(saw_code); assert!(saw_informal_turkish); Ok(()) } #[test] fn unmarked_code_detection_can_be_disabled_exactly() -> Result<(), super::TokenizerError> { let raw = b"foo_bar += 42".to_vec(); let detected = Tokenizer::embedded(TokenizerConfig::default())?.tokenize(raw.clone())?; let legacy = Tokenizer::embedded(TokenizerConfig { detect_unmarked_code: false, ..TokenizerConfig::default() })? .tokenize(raw.clone())?; assert_eq!(detected.decode(), raw); assert_eq!(legacy.decode(), raw); assert!(detected .units() .iter() .all(|unit| unit.mode == TokenMode::Code)); assert!(legacy .units() .iter() .any(|unit| unit.mode == TokenMode::Turkish)); Ok(()) } #[test] fn invalid_utf8_and_all_bytes_round_trip() -> Result<(), super::TokenizerError> { let tokenizer = Tokenizer::embedded(TokenizerConfig { mode: TokenizerMode::Turkish, ..TokenizerConfig::default() })?; let raw: Vec = (0_u8..=u8::MAX).collect(); let document = tokenizer.tokenize(raw.clone())?; assert_eq!(document.decode(), raw); assert!(document .units() .iter() .any(|unit| unit.status == TokenStatus::Opaque)); Ok(()) } #[test] fn runtime_proper_unknown_characters_align_without_losing_bytes( ) -> Result<(), super::TokenizerError> { let tokenizer = Tokenizer::embedded(TokenizerConfig::default())?; for (value, expected) in [("UŔK'A", "u?k"), ("UÁÑ'A", "uan"), ("Â", "a"), ("M1", "m1")] { let raw = value.as_bytes().to_vec(); let document = tokenizer.tokenize(raw.clone())?; assert_eq!(document.decode(), raw); assert!(document.units().iter().any(|unit| { unit.analysis .as_ref() .is_some_and(|analysis| analysis.canonical.to_lowercase().contains(expected)) })); } Ok(()) } #[test] fn punctuation_runs_are_split_for_exact_morphology() -> Result<(), super::TokenizerError> { let tokenizer = Tokenizer::embedded(TokenizerConfig::default())?; let raw = b"(. ... !?".to_vec(); let document = tokenizer.tokenize(raw.clone())?; assert_eq!(document.decode(), raw); let punctuation = document .units() .iter() .filter(|unit| unit.kind == nedo_core::LexicalKind::Punctuation) .collect::>(); assert_eq!(punctuation.len(), 7); assert!(punctuation .iter() .all(|unit| unit.span.end > unit.span.start)); Ok(()) } #[test] fn native_batch_preserves_order() -> Result<(), super::TokenizerError> { let tokenizer = Tokenizer::embedded(TokenizerConfig::default())?; let inputs = vec![b"bir".to_vec(), b"iki".to_vec(), b"uc".to_vec()]; let documents = tokenizer.tokenize_batch(&inputs, 2)?; assert_eq!(documents.len(), inputs.len()); for (document, input) in documents.iter().zip(inputs) { assert_eq!(document.decode(), input); } Ok(()) } #[test] fn batch_ranges_are_byte_weighted_and_cover_every_input() -> Result<(), super::TokenizerError> { let inputs = vec![ vec![0_u8; 700_000], vec![1_u8; 700_000], vec![2_u8; 700_000], vec![3_u8; 10], ]; let ranges = super::build_batch_ranges(&inputs, 2)?; assert_eq!(ranges, vec![0..2, 2..4]); Ok(()) } #[test] fn cached_batch_matches_independent_tokenization() -> Result<(), super::TokenizerError> { let tokenizer = Tokenizer::embedded(TokenizerConfig::default())?; let inputs = vec![ "Ankara'da evlerimizdekiler konuşuyor.".as_bytes().to_vec(), "Ankara'da evlerimizdekiler konuşuyor.".as_bytes().to_vec(), "Evlerimizdekiler Ankara'da konuşuyor.".as_bytes().to_vec(), ]; let cached = tokenizer.tokenize_batch(&inputs, 2)?; let independent = inputs .iter() .cloned() .map(|raw| tokenizer.tokenize(raw)) .collect::, _>>()?; assert_eq!(cached, independent); Ok(()) } #[test] fn dates_do_not_split_context_and_fallback_has_a_hard_limit( ) -> Result<(), super::TokenizerError> { let tokenizer = Tokenizer::embedded(TokenizerConfig { max_sentence_tokens: 512, max_fallback_chars: 8, ..TokenizerConfig::default() })?; let document = tokenizer.tokenize("23.07.2026 qwxqwxqwxqwxqwxqwxqwxqwx".as_bytes().to_vec())?; let date = document .units() .iter() .find(|unit| unit.kind == nedo_core::LexicalKind::Number) .ok_or(super::TokenizerError::InvalidConfiguration( "missing date unit", ))?; assert!(date.cuts.len() >= 4); let unknowns = document .units() .iter() .filter(|unit| unit.status == TokenStatus::Unknown) .collect::>(); assert!(unknowns.len() >= 2); for unit in &unknowns { let start = usize::try_from(unit.span.start) .map_err(|_| super::TokenizerError::LengthOverflow("test start"))?; let end = usize::try_from(unit.span.end) .map_err(|_| super::TokenizerError::LengthOverflow("test end"))?; assert!( std::str::from_utf8(&document.raw()[start..end]) .map_err(|_| super::TokenizerError::InvalidUtf8Unit)? .chars() .count() <= 8 ); } let pathological = "q".repeat(48 + 17); let tokenizer = Tokenizer::embedded(TokenizerConfig { mode: TokenizerMode::Turkish, ..TokenizerConfig::default() })?; let pathological_document = tokenizer.tokenize(pathological.as_bytes().to_vec())?; let pathological_unknowns = pathological_document .units() .iter() .filter(|unit| { matches!( unit.status, TokenStatus::Unknown | TokenStatus::Code | TokenStatus::Opaque ) }) .collect::>(); assert_eq!(pathological_unknowns.len(), 2); for unit in pathological_unknowns { let start = usize::try_from(unit.span.start) .map_err(|_| super::TokenizerError::LengthOverflow("test start"))?; let end = usize::try_from(unit.span.end) .map_err(|_| super::TokenizerError::LengthOverflow("test end"))?; assert!( std::str::from_utf8(&pathological_document.raw()[start..end]) .map_err(|_| super::TokenizerError::InvalidUtf8Unit)? .chars() .count() <= 48 ); } Ok(()) } #[test] fn completes_numeric_derivations_without_losing_apostrophe() -> Result<(), super::TokenizerError> { let tokenizer = Tokenizer::embedded(TokenizerConfig::default())?; for (raw, expected_id, expected_surface) in [ (b"20'nci".as_slice(), "Ord", "nci"), (b"1'incisini".as_slice(), "Ord", "inci"), (b"110'ar".as_slice(), "Dist", "ar"), (b"18'erli".as_slice(), "Dist", "er"), ("20'şer".as_bytes(), "Dist", "şer"), ("11’İNCİ".as_bytes(), "Ord", "inci"), ("1’İNCİSİNİ".as_bytes(), "Ord", "inci"), ("20’ŞER".as_bytes(), "Dist", "şer"), (b"I'inci".as_slice(), "Ord", "inci"), ("II’nci".as_bytes(), "Ord", "nci"), (b"IX'uncu".as_slice(), "Ord", "uncu"), ] { let document = tokenizer.tokenize(raw.to_vec())?; assert_eq!(document.decode(), raw); assert_eq!(document.units().len(), 1); let unit = &document.units()[0]; assert_eq!(unit.status, TokenStatus::Morphological); assert!(unit.cuts.len() >= 2); let analysis = unit .analysis .as_ref() .ok_or(super::TokenizerError::MissingAnalysis { index: 0 })?; assert!(analysis.morphemes.iter().any(|morpheme| { morpheme.id == expected_id && morpheme.surface == expected_surface })); } Ok(()) } #[test] fn no_context_uses_first_surface_valid_candidate_or_unknown( ) -> Result<(), super::TokenizerError> { for contextual_disambiguation in [true, false] { let tokenizer = Tokenizer::embedded(TokenizerConfig { contextual_disambiguation, ..TokenizerConfig::default() })?; for raw in ["Qʼumarkaj'da".as_bytes(), b"normal".as_slice()] { let document = tokenizer.tokenize(raw.to_vec())?; assert_eq!(document.decode(), raw); assert!(!document.units().is_empty()); } } Ok(()) } #[test] fn cached_batch_preserves_unicode_numeric_apostrophe_semantics( ) -> Result<(), super::TokenizerError> { let tokenizer = Tokenizer::embedded(TokenizerConfig::default())?; let inputs = [ "Alan 14.000 m²'den büyüktür.", "Debi 10 m³/s'dir.", "Payın ½’sini kullandı.", "FⅧ’ün devamı işlendi.", "Kapasite 1700m²'lik binadır.", "Muhtemel muhalefeti asgariye indirmenin altyapısını oluşturuyor.", ] .into_iter() .map(|value| value.as_bytes().to_vec()) .collect::>(); let cached = tokenizer.tokenize_batch(&inputs, 4)?; let independent = inputs .iter() .cloned() .map(|raw| tokenizer.tokenize(raw)) .collect::, _>>()?; assert_eq!(cached, independent); Ok(()) } #[test] fn flat_training_batch_matches_rich_reference_exactly() -> Result<(), super::TokenizerError> { let inputs = vec![ "Ankara'da evlerimizden mi? 20'nci, 3,14 ve 23.07.2026." .as_bytes() .to_vec(), b"```rust fn parseHttpRequest_header42() { return 0; } ```" .to_vec(), "Qʼumarkaj'da 🙂 snake_case camelCase".as_bytes().to_vec(), vec![0x00, 0xff, b'a', 0x80, b'Z'], "x".repeat(97).into_bytes(), ]; let newline_flags = vec![true, false, true, false, true]; let vocabulary = CharacterVocabulary::from_sorted(Vec::new())?; let option_sets = [ TrainingEncodingOptions::default(), TrainingEncodingOptions { emit_unit_boundaries: false, emit_morpheme_boundaries: true, emit_code_boundaries: true, }, TrainingEncodingOptions { emit_unit_boundaries: true, emit_morpheme_boundaries: false, emit_code_boundaries: false, }, ]; for mode in [ TokenizerMode::Auto, TokenizerMode::Turkish, TokenizerMode::Code, ] { for contextual_disambiguation in [true, false] { let tokenizer = Tokenizer::embedded(TokenizerConfig { mode, contextual_disambiguation, ..TokenizerConfig::default() })?; let rich = tokenizer.tokenize_batch(&inputs, 2)?; for options in option_sets { let reference = vocabulary.encode_training_batch(&rich, &newline_flags, options)?; let flat = tokenizer.encode_training_batch( &inputs, &newline_flags, &vocabulary, 2, options, )?; assert_eq!(flat, reference); } } } Ok(()) } #[test] fn stateful_flat_encoder_reuses_cache_without_changing_output( ) -> Result<(), super::TokenizerError> { let tokenizer = Tokenizer::embedded(TokenizerConfig::default())?; let vocabulary = CharacterVocabulary::from_sorted(Vec::new())?; let inputs = vec![ "Evlerimizden geldik, evlerimizden çıktık." .as_bytes() .to_vec(), "Evlerimizden geldik, evlerimizden çıktık." .as_bytes() .to_vec(), ]; let newlines = vec![false, true]; let mut encoder = tokenizer.training_encoder(&vocabulary, 1, TrainingEncodingOptions::default())?; let first = encoder.encode_batch(&inputs, &newlines)?; let first_stats = encoder.cache_stats(); let second = encoder.encode_batch(&inputs, &newlines)?; let second_stats = encoder.cache_stats(); assert_eq!(first, second); assert!(second_stats.hits > first_stats.hits); assert_eq!(second_stats.misses, first_stats.misses); assert!(second_stats.entries > 0); assert!(second_stats.approximate_bytes > 0); encoder.clear_caches(); assert_eq!(encoder.cache_stats(), super::TrainingCacheStats::default()); Ok(()) } #[test] fn persistent_surface_runtime_matches_stateless_and_reuses_cache( ) -> Result<(), super::TokenizerError> { let tokenizer = Tokenizer::embedded(TokenizerConfig { mode: TokenizerMode::Turkish, ..TokenizerConfig::default() })?; let vocabulary = SurfaceVocabulary::from_ranked(Vec::new())?; let inputs = vec![ "Evlerimizden geldik, evlerimizden çıktık." .as_bytes() .to_vec(), "Evlerimizden geldik, evlerimizden çıktık." .as_bytes() .to_vec(), ]; let newlines = vec![false, false]; let expected = tokenizer.encode_surface_batch(&inputs, &newlines, &vocabulary, 1, true)?; let mut runtime = tokenizer.surface_runtime_cache(1, SurfaceEncoderOptions::one_pass(true))?; let first = tokenizer.encode_surface_batch_with_runtime( &inputs, &newlines, &vocabulary, &mut runtime, )?; let first_stats = runtime.cache_stats(); let second = tokenizer.encode_surface_batch_with_runtime( &inputs, &newlines, &vocabulary, &mut runtime, )?; let second_stats = runtime.cache_stats(); assert_eq!(first, expected); assert_eq!(second, expected); assert!(second_stats.hits > first_stats.hits); assert_eq!(second_stats.misses, first_stats.misses); runtime.clear(); assert_eq!(runtime.cache_stats(), super::TrainingCacheStats::default()); Ok(()) } #[test] fn persistent_surface_runtime_rejects_different_tokenizer_config( ) -> Result<(), super::TokenizerError> { let turkish = Tokenizer::embedded(TokenizerConfig { mode: TokenizerMode::Turkish, ..TokenizerConfig::default() })?; let code = Tokenizer::embedded(TokenizerConfig { mode: TokenizerMode::Code, ..TokenizerConfig::default() })?; let vocabulary = SurfaceVocabulary::from_ranked(Vec::new())?; let mut runtime = turkish.surface_runtime_cache(1, SurfaceEncoderOptions::one_pass(true))?; let inputs = vec![b"evlerimizden".to_vec()]; let newlines = vec![false]; let error = code .encode_surface_batch_with_runtime(&inputs, &newlines, &vocabulary, &mut runtime) .expect_err("runtime must be bound to the tokenizer configuration that created it"); assert!(matches!( error, super::TokenizerError::InvalidConfiguration(_) )); Ok(()) } #[test] fn operators_split_only_in_turkish_mode() -> Result<(), super::TokenizerError> { let turkish = Tokenizer::embedded(TokenizerConfig::default())?; let turkish_document = turkish.tokenize(b"?.".to_vec())?; assert_eq!(turkish_document.decode(), b"?."); assert_eq!(turkish_document.units().len(), 2); let surfaces = turkish_document .units() .iter() .map(|unit| { let start = usize::try_from(unit.span.start) .map_err(|_| super::TokenizerError::LengthOverflow("test start"))?; let end = usize::try_from(unit.span.end) .map_err(|_| super::TokenizerError::LengthOverflow("test end"))?; Ok(&turkish_document.raw()[start..end]) }) .collect::, super::TokenizerError>>()?; assert_eq!(surfaces, [b"?".as_slice(), b".".as_slice()]); let code = Tokenizer::embedded(TokenizerConfig { mode: TokenizerMode::Code, ..TokenizerConfig::default() })?; let code_document = code.tokenize(b"?.".to_vec())?; assert_eq!(code_document.decode(), b"?."); assert_eq!(code_document.units().len(), 1); assert_eq!(code_document.units()[0].status, TokenStatus::Code); Ok(()) } } /// Read-only diagnostics used to build and verify compiled surface tables. #[cfg(feature = "surface-audit")] pub mod surface_audit { use super::{ build_flat_analysis_set, explicit_code_spans, scan, split_units, unit_str, LexicalKind, TokenMode, Tokenizer, TokenizerError, TokenizerMode, }; /// One distinct tokenizer-visible output among a surface form's analyses. #[derive(Clone, Debug, Eq, PartialEq)] pub struct SurfaceOutputClassAudit { /// Relative byte cuts produced by this class. pub cuts: Vec, /// Whether this class represents the explicit unknown analysis. pub unknown: bool, /// Number of native analyses collapsed into this output class. pub analysis_count: usize, } /// Exact candidate and output-class audit for one surface form. #[derive(Clone, Debug, Eq, PartialEq)] pub struct SurfaceAnalysisAudit { /// Number of surface-valid native analysis candidates. pub candidate_count: usize, /// Distinct tokenizer-visible `(cuts, unknown)` outputs. pub output_classes: Vec, } impl SurfaceAnalysisAudit { /// Whether context can never change this surface form's tokenizer output. #[must_use] pub fn output_invariant(&self) -> bool { self.output_classes.len() == 1 } } impl Tokenizer<'_> { /// Returns the exact surface strings that enter native morphology for one document. /// /// # Errors /// /// Returns an error if scanning, unit splitting, or UTF-8 extraction fails. pub fn audit_surface_tokens(&self, raw: Vec) -> Result, TokenizerError> { Ok(self .audit_surface_segments(raw)? .into_iter() .flatten() .collect()) } /// Returns exact sentence/chunk segments entering contextual morphology. /// /// # Errors /// /// Returns an error if scanning, unit splitting, UTF-8 extraction, or boundary detection fails. pub fn audit_surface_segments( &self, raw: Vec, ) -> Result>, TokenizerError> { let scan = scan(raw)?; let decoded = scan.document().decode(); let code_spans = match self.config.mode { TokenizerMode::Auto => explicit_code_spans(decoded), TokenizerMode::Turkish | TokenizerMode::Code => Vec::new(), }; let units = split_units(&scan, &code_spans, self.config.mode)?; let mut output = Vec::new(); let mut segment = Vec::new(); for unit in &units { if unit.mode != TokenMode::Turkish || matches!( unit.kind, LexicalKind::LineBreak | LexicalKind::Control | LexicalKind::Opaque ) { if !segment.is_empty() { output.push(std::mem::take(&mut segment)); } continue; } if matches!(unit.kind, LexicalKind::Whitespace) { continue; } segment.push(unit_str(decoded, unit.span)?.to_owned()); let boundary = super::is_sentence_boundary(unit.kind, decoded, unit.span)?; if boundary || segment.len() >= self.config.max_sentence_tokens { output.push(std::mem::take(&mut segment)); } } if !segment.is_empty() { output.push(segment); } Ok(output) } /// Builds one exact full-table compiler entry in production candidate order. /// /// # Errors /// /// Returns an error if native analysis, alignment, or candidate cardinality fails. #[cfg(feature = "compiled-surface-table")] pub fn audit_compiled_surface_analysis_entry( &self, token: &str, ) -> Result { let analyses = super::analyze_token_with_quality_fallback(&self.morphology, token)?; let set = build_flat_analysis_set(token, analyses)?; if set.relative_cuts.len() != set.ambiguity.len() || set.ambiguity.len() != set.unknown.len() { return Err(TokenizerError::InvalidTrainingEncoding( "compiled audit candidate cardinalities differ", )); } let candidates = set .relative_cuts .iter() .zip(&set.ambiguity) .zip(&set.unknown) .map( |((cuts, ambiguity), unknown)| super::CompiledSurfaceCandidateEntry { cuts: cuts.clone(), unknown: *unknown, canonical: ambiguity.canonical.clone(), lemma: ambiguity.lemma.clone(), igs: ambiguity.igs.clone(), java_hash: ambiguity.java_hash, }, ) .collect(); Ok(super::CompiledSurfaceAnalysisEntry { surface: token.as_bytes().to_vec(), candidates, }) } /// Builds one exact NedoFormer shadow-analysis table entry. /// /// # Errors /// /// Returns an error if shadow analysis, alignment, or candidate cardinality fails. #[cfg(feature = "compiled-surface-table")] pub fn audit_nedoformer_compiled_surface_analysis_entry( &self, token: &str, ) -> Result { let analyses = super::analyze_token_with_nedoformer_shadow(&self.morphology, token)?; let set = build_flat_analysis_set(token, analyses)?; if set.relative_cuts.len() != set.ambiguity.len() || set.ambiguity.len() != set.unknown.len() { return Err(TokenizerError::InvalidTrainingEncoding( "NedoFormer compiled audit candidate cardinalities differ", )); } let candidates = set .relative_cuts .iter() .zip(&set.ambiguity) .zip(&set.unknown) .map( |((cuts, ambiguity), unknown)| super::CompiledSurfaceCandidateEntry { cuts: cuts.clone(), unknown: *unknown, canonical: ambiguity.canonical.clone(), lemma: ambiguity.lemma.clone(), igs: ambiguity.igs.clone(), java_hash: ambiguity.java_hash, }, ) .collect(); Ok(super::CompiledSurfaceAnalysisEntry { surface: token.as_bytes().to_vec(), candidates, }) } /// Audits all surface-valid analyses using the exact production alignment path. /// /// # Errors /// /// Returns an error if native analysis or exact surface alignment fails. pub fn audit_surface_analysis( &self, token: &str, ) -> Result { let analyses = self.morphology.analyze_token(token)?; let set = build_flat_analysis_set(token, analyses)?; let mut output_classes = Vec::::new(); for (cuts, unknown) in set.relative_cuts.iter().zip(&set.unknown) { if let Some(existing) = output_classes.iter_mut().find(|class| { class.cuts.as_slice() == cuts.as_slice() && class.unknown == *unknown }) { existing.analysis_count = existing.analysis_count.saturating_add(1); } else { output_classes.push(SurfaceOutputClassAudit { cuts: cuts.to_vec(), unknown: *unknown, analysis_count: 1, }); } } Ok(SurfaceAnalysisAudit { candidate_count: set.relative_cuts.len(), output_classes, }) } } } #[cfg(all(test, feature = "surface-audit", feature = "compiled-surface-table"))] mod compiled_surface_safety_tests { use super::{ encode_compiled_surface_analysis_table, CompiledSurfaceAnalysisTable, SurfaceEncoderOptions, SurfaceVocabulary, Tokenizer, TokenizerConfig, TokenizerError, }; #[test] fn semantic_verification_rejects_valid_checksum_with_wrong_cuts() { let exact_tokenizer = Tokenizer::embedded(TokenizerConfig::default()).expect("embedded tokenizer must load"); let mut entry = exact_tokenizer .audit_compiled_surface_analysis_entry("evler") .expect("exact compiler entry must build"); let first = entry .candidates .first_mut() .expect("surface must have at least one candidate"); first.cuts = if first.cuts == [1] { vec![2] } else { vec![1] }; let bytes = encode_compiled_surface_analysis_table(&[entry]) .expect("wrong-but-structurally-valid table must encode with a valid checksum"); let table = CompiledSurfaceAnalysisTable::from_bytes(&bytes) .expect("wrong-but-checksummed table must parse structurally"); let verifier = Tokenizer::embedded(TokenizerConfig::default()).expect("embedded tokenizer must load"); match verifier.with_verified_compiled_surface_analysis_table(table) { Err(TokenizerError::InvalidCompiledSurfaceTable { reason, .. }) => { assert_eq!(reason, "candidate cuts or candidate order differ"); } Err(other) => panic!("unexpected verification error: {other}"), Ok(_) => panic!("semantic verification accepted deliberately wrong cuts"), } } #[test] fn semantic_verification_accepts_exact_compiler_output() { let compiler = Tokenizer::embedded(TokenizerConfig::default()).expect("embedded tokenizer must load"); let entry = compiler .audit_compiled_surface_analysis_entry("evler") .expect("exact compiler entry must build"); let bytes = encode_compiled_surface_analysis_table(&[entry]).expect("exact table must encode"); let table = CompiledSurfaceAnalysisTable::from_bytes(&bytes).expect("exact table must parse"); let verifier = Tokenizer::embedded(TokenizerConfig::default()).expect("embedded tokenizer must load"); assert!(verifier .with_verified_compiled_surface_analysis_table(table) .is_ok()); } #[test] fn partial_compiled_table_preserves_surface_output_and_reduces_live_misses() { let config = TokenizerConfig { mode: super::TokenizerMode::Turkish, ..TokenizerConfig::default() }; let compiler = Tokenizer::embedded(config).expect("embedded tokenizer must load"); let entry = compiler .audit_compiled_surface_analysis_entry("evlerimizden") .expect("partial compiler entry must build"); let bytes = encode_compiled_surface_analysis_table(&[entry]).expect("partial table must encode"); let table = CompiledSurfaceAnalysisTable::from_bytes(&bytes).expect("partial table must parse"); let compiled = Tokenizer::embedded(config) .expect("embedded tokenizer must load") .with_verified_compiled_surface_analysis_table(table) .expect("partial table must verify"); let baseline = Tokenizer::embedded(config).expect("embedded tokenizer must load"); let vocabulary = SurfaceVocabulary::from_ranked(Vec::new()) .expect("byte-fallback vocabulary must build"); let inputs = vec![ "evlerimizden bugün geldik.".as_bytes().to_vec(), "evlerimizden yarın çıkacağız.".as_bytes().to_vec(), ]; let newline_flags = vec![false; inputs.len()]; let mut baseline_encoder = baseline .surface_encoder_with_options(&vocabulary, 1, SurfaceEncoderOptions::one_pass(true)) .expect("baseline encoder must build"); let expected = baseline_encoder .encode_batch(&inputs, &newline_flags) .expect("baseline batch must encode"); let baseline_misses = baseline_encoder.cache_stats().misses; let mut compiled_encoder = compiled .surface_encoder_with_options(&vocabulary, 1, SurfaceEncoderOptions::one_pass(true)) .expect("compiled encoder must build"); let actual = compiled_encoder .encode_batch(&inputs, &newline_flags) .expect("compiled batch must encode"); assert_eq!(actual, expected); assert!(compiled_encoder.cache_stats().misses < baseline_misses); } #[test] fn pinned_nedoformer_table_requires_independent_exact_digest() { let compiler = Tokenizer::embedded(TokenizerConfig::default()).expect("embedded tokenizer must load"); let entry = compiler .audit_nedoformer_compiled_surface_analysis_entry("cocuklarimizdan") .expect("NedoFormer shadow entry must compile"); let bytes = encode_compiled_surface_analysis_table(&[entry]) .expect("NedoFormer shadow table must encode"); let table = CompiledSurfaceAnalysisTable::from_bytes(&bytes) .expect("NedoFormer shadow table must parse"); let digest = table.digest(); let mut wrong = digest; wrong[0] ^= 1; let verifier = Tokenizer::embedded(TokenizerConfig::default()).expect("embedded tokenizer must load"); assert!(matches!( verifier.with_pinned_nedoformer_compiled_surface_analysis_table(table, wrong), Err(TokenizerError::CompiledSurfaceTableSealMismatch { .. }) )); let table = CompiledSurfaceAnalysisTable::from_bytes(&bytes) .expect("NedoFormer shadow table must parse twice"); let pinned = Tokenizer::embedded(TokenizerConfig::default()) .expect("embedded tokenizer must load") .with_pinned_nedoformer_compiled_surface_analysis_table(table, digest) .expect("trusted exact digest must permit fast attach"); let input = vec![b"cocuklarimizdan geliyor mu?".to_vec()]; let baseline = Tokenizer::embedded(TokenizerConfig::default()).expect("embedded tokenizer must load"); assert_eq!( pinned .nedoformer_sidecar_batch(&input, 1) .expect("pinned sidecar must encode"), baseline .nedoformer_sidecar_batch(&input, 1) .expect("baseline sidecar must encode") ); } #[test] fn nedoformer_shadow_table_is_separately_verified_and_sidecar_exact() { let compiler = Tokenizer::embedded(TokenizerConfig::default()).expect("embedded tokenizer must load"); let entry = compiler .audit_nedoformer_compiled_surface_analysis_entry("cocuklarimizdan") .expect("NedoFormer shadow entry must compile"); let bytes = encode_compiled_surface_analysis_table(&[entry]) .expect("NedoFormer shadow table must encode"); let normal_table = CompiledSurfaceAnalysisTable::from_bytes(&bytes) .expect("NedoFormer shadow table must parse"); let normal = Tokenizer::embedded(TokenizerConfig::default()).expect("embedded tokenizer must load"); assert!(normal .with_verified_compiled_surface_analysis_table(normal_table) .is_err()); let ndf_table = CompiledSurfaceAnalysisTable::from_bytes(&bytes) .expect("NedoFormer shadow table must parse twice"); let baseline = Tokenizer::embedded(TokenizerConfig::default()).expect("embedded tokenizer must load"); let accelerated = Tokenizer::embedded(TokenizerConfig::default()) .expect("embedded tokenizer must load") .with_verified_nedoformer_compiled_surface_analysis_table(ndf_table) .expect("NedoFormer verifier must accept exact shadow table"); let input = vec![b"cocuklarimizdan geliyor mu?".to_vec()]; assert_eq!( baseline .nedoformer_sidecar_batch(&input, 1) .expect("baseline sidecar must encode"), accelerated .nedoformer_sidecar_batch(&input, 1) .expect("compiled NedoFormer sidecar must encode") ); } }