Spaces:
Running on Zero
Running on Zero
| use aho_corasick::{AhoCorasick, AhoCorasickBuilder, AhoCorasickKind, MatchKind}; | |
| use pyo3::exceptions::PyValueError; | |
| use pyo3::prelude::*; | |
| use rayon::prelude::*; | |
| use rayon::ThreadPoolBuilder; | |
| use std::collections::HashMap; | |
| static UNICODE_INDEX: &[u8] = include_bytes!("generated/unicode_index.bin"); | |
| static UNICODE_POOL: &[u8] = include_bytes!("generated/unicode_pool.bin"); | |
| static PYTHON_WHITESPACE: &[u8] = include_bytes!("generated/python_whitespace.bin"); | |
| const ALIAS: u8 = 0; | |
| const LEXICAL: u8 = 1; | |
| const CONJUNCTION: u8 = 2; | |
| const FAMILY_NONE: i8 = -1; | |
| const FAMILY_SP500: i8 = 0; | |
| const FAMILY_NASDAQ100: i8 = 1; | |
| const FAMILY_RUSSELL2000: i8 = 2; | |
| const FAMILY_MARKET_WIDE: i8 = 3; | |
| const ARTICLE_ACCEPTED: u8 = 0; | |
| const ARTICLE_REJECTED: u8 = 2; | |
| const EXP_SUPPORTED: u8 = 0; | |
| const EXP_PLAUSIBLE: u8 = 1; | |
| const EXP_NOT_SUPPORTED: u8 = 2; | |
| const NORMALIZED_DFA_MAX_PATTERNS: usize = 512; | |
| const NORMALIZED_DFA_MAX_PATTERN_BYTES: usize = 16 * 1024; | |
| const NORMALIZED_DFA_MAX_HEAP_BYTES: usize = 32 * 1024 * 1024; | |
| fn try_standard_automaton( | |
| patterns: &[String], | |
| ascii_case_insensitive: bool, | |
| kind: Option<AhoCorasickKind>, | |
| ) -> Result<AhoCorasick, String> { | |
| if patterns.iter().any(|pattern| pattern.is_empty()) { | |
| return Err("patterns must be non-empty".to_string()); | |
| } | |
| let mut builder = AhoCorasickBuilder::new(); | |
| builder | |
| .match_kind(MatchKind::Standard) | |
| .ascii_case_insensitive(ascii_case_insensitive) | |
| .prefilter(true); | |
| if let Some(kind) = kind { | |
| builder.kind(Some(kind)); | |
| } | |
| builder.build(patterns).map_err(|error| error.to_string()) | |
| } | |
| fn build_auto_automaton( | |
| patterns: &[String], | |
| ascii_case_insensitive: bool, | |
| ) -> PyResult<AhoCorasick> { | |
| try_standard_automaton(patterns, ascii_case_insensitive, None) | |
| .map_err(PyValueError::new_err) | |
| } | |
| fn normalized_dfa_eligible(patterns: &[String]) -> bool { | |
| patterns.len() <= NORMALIZED_DFA_MAX_PATTERNS | |
| && patterns | |
| .iter() | |
| .try_fold(0usize, |total, pattern| total.checked_add(pattern.len())) | |
| .is_some_and(|total| total <= NORMALIZED_DFA_MAX_PATTERN_BYTES) | |
| } | |
| fn build_automaton(patterns: &[String]) -> PyResult<AhoCorasick> { | |
| if normalized_dfa_eligible(patterns) { | |
| if let Ok(dfa) = | |
| try_standard_automaton(patterns, false, Some(AhoCorasickKind::DFA)) | |
| { | |
| if dfa.memory_usage() <= NORMALIZED_DFA_MAX_HEAP_BYTES { | |
| return Ok(dfa); | |
| } | |
| } | |
| } | |
| build_auto_automaton(patterns, false) | |
| } | |
| struct ConsumerSpec { | |
| kind: u8, | |
| a: usize, | |
| b: usize, | |
| c: usize, | |
| } | |
| enum ExactMode { | |
| CaseSensitive, | |
| PythonIgnoreCaseAsciiLiteral, | |
| } | |
| struct ExactSpec { | |
| value: Vec<char>, | |
| mode: ExactMode, | |
| kind: u8, | |
| index: usize, | |
| } | |
| struct ExactAutomaton { | |
| automaton: AhoCorasick, | |
| pattern_specs: Vec<Vec<usize>>, | |
| } | |
| struct ExactSpecs { | |
| specs: Vec<ExactSpec>, | |
| case_sensitive: Option<ExactAutomaton>, | |
| ignorecase_ascii: Option<ExactAutomaton>, | |
| } | |
| struct ExactByteMatch { | |
| spec_index: usize, | |
| start: usize, | |
| end: usize, | |
| } | |
| struct ContextRule { | |
| window_chars: usize, | |
| accept_clauses: Vec<Vec<Vec<u8>>>, | |
| reject_terms: Vec<Vec<u8>>, | |
| } | |
| struct AliasSpec { | |
| canonical_group: usize, | |
| context_rule: Option<usize>, | |
| valid_from_epoch: i64, | |
| valid_to_epoch: Option<i64>, | |
| family: i8, | |
| entity_kind: u8, | |
| unresolved_tag: bool, | |
| } | |
| struct LexicalSpec { | |
| context_window_chars: usize, | |
| context_terms: Vec<Vec<u8>>, | |
| is_event: bool, | |
| market_wide_authorized: bool, | |
| unresolved_tag: bool, | |
| has_event_family: bool, | |
| } | |
| struct ConjunctionSpec { | |
| actor_phrase_count: usize, | |
| action_phrase_count: usize, | |
| object_phrase_count: usize, | |
| scope_phrase_count: usize, | |
| } | |
| impl ConjunctionSpec { | |
| fn contains_phrase(&self, field: usize, phrase_index: usize) -> bool { | |
| let count = match field { | |
| 0 => self.actor_phrase_count, | |
| 1 => self.action_phrase_count, | |
| 2 => self.object_phrase_count, | |
| 3 => self.scope_phrase_count, | |
| _ => return false, | |
| }; | |
| phrase_index < count | |
| } | |
| } | |
| struct ConjunctionMarks { | |
| generation: u32, | |
| stamps: Vec<u32>, | |
| masks: Vec<u8>, | |
| touched: Vec<usize>, | |
| } | |
| impl ConjunctionMarks { | |
| fn begin_row(&mut self, rule_count: usize) { | |
| if self.stamps.len() < rule_count { | |
| self.stamps.resize(rule_count, 0); | |
| self.masks.resize(rule_count, 0); | |
| } | |
| self.touched.clear(); | |
| self.generation = self.generation.wrapping_add(1); | |
| if self.generation == 0 { | |
| self.stamps.fill(0); | |
| self.generation = 1; | |
| } | |
| } | |
| fn mark(&mut self, rule_index: usize, field: usize) { | |
| if rule_index >= self.stamps.len() || field >= 4 { | |
| return; | |
| } | |
| if self.stamps[rule_index] != self.generation { | |
| self.stamps[rule_index] = self.generation; | |
| self.masks[rule_index] = 0; | |
| self.touched.push(rule_index); | |
| } | |
| self.masks[rule_index] |= 1u8 << field; | |
| } | |
| fn is_complete(&self, rule_index: usize) -> bool { | |
| self.stamps[rule_index] == self.generation && self.masks[rule_index] == 0b1111 | |
| } | |
| fn is_empty(&self) -> bool { | |
| self.touched.is_empty() | |
| } | |
| } | |
| // (row_index, article_status, sp500, nasdaq100, russell2000, market_wide, | |
| // observation_count, native_candidate, exact_candidate) | |
| type FactRow = (usize, u8, u8, u8, u8, u8, usize, bool, bool); | |
| struct ScanScratch { | |
| chars: Vec<char>, | |
| normalized: Vec<u8>, | |
| offsets: Vec<usize>, | |
| exact_byte_matches: Vec<ExactByteMatch>, | |
| exact_matches: Vec<(usize, usize, usize)>, | |
| exact_boundaries: Vec<usize>, | |
| alias_candidates: Vec<(usize, usize, usize)>, | |
| lexical_candidates: Vec<(usize, usize, usize)>, | |
| conjunction_marks: ConjunctionMarks, | |
| } | |
| impl ScanScratch { | |
| fn reset(&mut self, conjunction_count: usize) { | |
| self.chars.clear(); | |
| self.normalized.clear(); | |
| self.offsets.clear(); | |
| self.exact_byte_matches.clear(); | |
| self.exact_matches.clear(); | |
| self.exact_boundaries.clear(); | |
| self.alias_candidates.clear(); | |
| self.lexical_candidates.clear(); | |
| self.conjunction_marks.begin_row(conjunction_count); | |
| } | |
| } | |
| fn is_python_ascii_space(byte: u8) -> bool { | |
| matches!( | |
| byte, | |
| b' ' | b'\t' | b'\n' | 0x0b | 0x0c | b'\r' | 0x1c..=0x1f | |
| ) | |
| } | |
| fn is_separator_ascii(byte: u8) -> bool { | |
| matches!( | |
| byte, | |
| b'-' | b'.' | b',' | b':' | b';' | b'(' | b')' | b'[' | b']' | b'{' | b'}' | b'/' | |
| ) | |
| } | |
| fn is_token_byte(byte: u8) -> bool { | |
| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'_' | |
| } | |
| fn is_exact_token_char(ch: char, mode: ExactMode) -> bool { | |
| if ch.is_ascii_alphanumeric() || ch == '_' { | |
| return true; | |
| } | |
| // Python's Unicode-aware re.IGNORECASE expands [A-Za-z] to these four | |
| // additional code points. They are token characters only for the | |
| // python_ignorecase_ascii_literal mode; applying this globally would change | |
| // the frozen boundary behavior of case-sensitive exact aliases. | |
| matches!(mode, ExactMode::PythonIgnoreCaseAsciiLiteral) | |
| && matches!(ch, '\u{0130}' | '\u{0131}' | '\u{017f}' | '\u{212a}') | |
| } | |
| fn is_python_space(ch: char) -> bool { | |
| let cp = ch as usize; | |
| let byte = PYTHON_WHITESPACE[cp >> 3]; | |
| (byte & (1 << (cp & 7))) != 0 | |
| } | |
| fn mapped_bytes(ch: char) -> Option<&'static [u8]> { | |
| let cp = ch as usize; | |
| let base = cp * 4; | |
| let offset = u32::from_le_bytes([ | |
| UNICODE_INDEX[base], | |
| UNICODE_INDEX[base + 1], | |
| UNICODE_INDEX[base + 2], | |
| UNICODE_INDEX[base + 3], | |
| ]) as usize; | |
| if offset == 0 { | |
| return None; | |
| } | |
| let len = u16::from_le_bytes([UNICODE_POOL[offset], UNICODE_POOL[offset + 1]]) as usize; | |
| Some(&UNICODE_POOL[offset + 2..offset + 2 + len]) | |
| } | |
| fn normalize_post_char( | |
| ch: char, | |
| source_index: usize, | |
| normalized: &mut Vec<u8>, | |
| offsets: &mut Vec<usize>, | |
| pending_space: &mut bool, | |
| ) { | |
| let ch = match ch { | |
| '-' | '‐' | '‑' | '‒' | '–' | '—' | '―' | '−' => ' ', | |
| '.' | ',' | ':' | ';' | '(' | ')' | '[' | ']' | '{' | '}' | '／' | '/' => ' ', | |
| '\'' | '’' | '‘' | '‛' | '`' | '´' => '\'', | |
| other => other, | |
| }; | |
| if is_python_space(ch) { | |
| *pending_space = !normalized.is_empty(); | |
| return; | |
| } | |
| if *pending_space { | |
| normalized.push(b' '); | |
| offsets.push(source_index); | |
| *pending_space = false; | |
| } | |
| let mut buffer = [0u8; 4]; | |
| let encoded = ch.encode_utf8(&mut buffer).as_bytes(); | |
| normalized.extend_from_slice(encoded); | |
| offsets.extend(std::iter::repeat(source_index).take(encoded.len())); | |
| } | |
| fn normalize_source_char( | |
| source_character: char, | |
| source_index: usize, | |
| normalized: &mut Vec<u8>, | |
| offsets: &mut Vec<usize>, | |
| pending_space: &mut bool, | |
| ) { | |
| if source_character.is_ascii() { | |
| let mut byte = (source_character as u8).to_ascii_lowercase(); | |
| if is_separator_ascii(byte) { | |
| byte = b' '; | |
| } else if byte == b'`' { | |
| byte = b'\''; | |
| } | |
| if is_python_ascii_space(byte) { | |
| *pending_space = !normalized.is_empty(); | |
| return; | |
| } | |
| if *pending_space { | |
| normalized.push(b' '); | |
| offsets.push(source_index); | |
| *pending_space = false; | |
| } | |
| normalized.push(byte); | |
| offsets.push(source_index); | |
| } else if let Some(mapped) = mapped_bytes(source_character) { | |
| let mapped = std::str::from_utf8(mapped).expect("generated Unicode pool is valid UTF-8"); | |
| for character in mapped.chars() { | |
| normalize_post_char(character, source_index, normalized, offsets, pending_space); | |
| } | |
| } else { | |
| normalize_post_char( | |
| source_character, | |
| source_index, | |
| normalized, | |
| offsets, | |
| pending_space, | |
| ); | |
| } | |
| } | |
| fn normalize_chars_into( | |
| chars: &[char], | |
| source_base: usize, | |
| normalized: &mut Vec<u8>, | |
| offsets: &mut Vec<usize>, | |
| ) { | |
| normalized.clear(); | |
| offsets.clear(); | |
| normalized.reserve(chars.len()); | |
| offsets.reserve(chars.len()); | |
| let mut pending_space = false; | |
| for (local_index, source_character) in chars.iter().copied().enumerate() { | |
| let source_index = source_base + local_index; | |
| normalize_source_char( | |
| source_character, | |
| source_index, | |
| normalized, | |
| offsets, | |
| &mut pending_space, | |
| ); | |
| } | |
| } | |
| fn normalize_ascii_byte(mut byte: u8) -> u8 { | |
| byte.make_ascii_lowercase(); | |
| if is_separator_ascii(byte) { | |
| b' ' | |
| } else if byte == b'`' { | |
| b'\'' | |
| } else { | |
| byte | |
| } | |
| } | |
| fn normalize_ascii_text_into(text: &str, normalized: &mut Vec<u8>, offsets: &mut Vec<usize>) { | |
| debug_assert!(text.is_ascii()); | |
| let mut pending_space = false; | |
| for (source_index, source_byte) in text.bytes().enumerate() { | |
| let byte = normalize_ascii_byte(source_byte); | |
| if is_python_ascii_space(byte) { | |
| pending_space = !normalized.is_empty(); | |
| continue; | |
| } | |
| if pending_space { | |
| normalized.push(b' '); | |
| offsets.push(source_index); | |
| pending_space = false; | |
| } | |
| normalized.push(byte); | |
| offsets.push(source_index); | |
| } | |
| } | |
| // Phase31A mixed-row block candidate; whole-ASCII control is unchanged. | |
| // Derived from a2dbcf9's Phase26 transform and scalar collapse-mask emitter. | |
| // The caller retains the frozen Unicode normalizer and whole-ASCII control. | |
| fn mixed_ascii_avx2_available() -> bool { | |
| static AVAILABLE: std::sync::OnceLock<bool> = std::sync::OnceLock::new(); | |
| *AVAILABLE.get_or_init(|| std::arch::is_x86_feature_detected!("avx2")) | |
| } | |
| const MIXED_ASCII_COLLAPSE_BY_LOW_NIBBLE: [u8; 16] = | |
| [4, 0, 0, 0, 0, 0, 0, 0, 4, 5, 9, 169, 7, 167, 6, 6]; | |
| const MIXED_ASCII_HIGH_NIBBLE_BITS: [u8; 16] = | |
| [1, 2, 4, 8, 16, 32, 64, 128, 0, 0, 0, 0, 0, 0, 0, 0]; | |
| fn emit_mixed_ascii_block( | |
| mapped: &[u8], | |
| collapse_mask: u32, | |
| source_base: usize, | |
| normalized: &mut Vec<u8>, | |
| offsets: &mut Vec<usize>, | |
| pending_space: &mut bool, | |
| ) { | |
| debug_assert!(mapped.len() <= 32); | |
| let mut local_index = 0usize; | |
| while local_index < mapped.len() { | |
| let remaining_mask = collapse_mask >> local_index; | |
| let remaining_len = mapped.len() - local_index; | |
| if remaining_mask & 1 != 0 { | |
| let run_len = (remaining_mask.trailing_ones() as usize).min(remaining_len); | |
| *pending_space = !normalized.is_empty(); | |
| local_index += run_len; | |
| continue; | |
| } | |
| let run_len = (remaining_mask.trailing_zeros() as usize).min(remaining_len); | |
| if *pending_space { | |
| normalized.push(b' '); | |
| offsets.push(source_base + local_index); | |
| *pending_space = false; | |
| } | |
| let run_end = local_index + run_len; | |
| normalized.extend_from_slice(&mapped[local_index..run_end]); | |
| offsets.extend(source_base + local_index..source_base + run_end); | |
| local_index = run_end; | |
| } | |
| } | |
| unsafe fn normalize_mixed_ascii_prefix_avx2_into( | |
| bytes: &[u8], | |
| source_base: usize, | |
| normalized: &mut Vec<u8>, | |
| offsets: &mut Vec<usize>, | |
| pending_space: &mut bool, | |
| ) -> usize { | |
| use std::arch::x86_64::*; | |
| // Every load is within the provided slice. A vector may contain UTF-8, | |
| // but only the ASCII prefix before its first high byte may be emitted. | |
| // The caller checks AVX2 support before entering this target-feature body. | |
| let low_table = _mm256_broadcastsi128_si256(unsafe { | |
| _mm_loadu_si128(MIXED_ASCII_COLLAPSE_BY_LOW_NIBBLE.as_ptr().cast()) | |
| }); | |
| let high_table = _mm256_broadcastsi128_si256(unsafe { | |
| _mm_loadu_si128(MIXED_ASCII_HIGH_NIBBLE_BITS.as_ptr().cast()) | |
| }); | |
| let low_nibble_mask = _mm256_set1_epi8(0x0f); | |
| let lowercase_bit = _mm256_set1_epi8(0x20); | |
| let upper_lower_bound = _mm256_set1_epi8((b'A' - 1) as i8); | |
| let upper_upper_bound = _mm256_set1_epi8((b'Z' + 1) as i8); | |
| let backtick = _mm256_set1_epi8(b'`' as i8); | |
| let apostrophe = _mm256_set1_epi8(b'\'' as i8); | |
| let mut mapped = [0u8; 32]; | |
| let mut consumed = 0usize; | |
| while bytes.len() - consumed >= 32 { | |
| let source = unsafe { _mm256_loadu_si256(bytes.as_ptr().add(consumed).cast()) }; | |
| let high_mask = _mm256_movemask_epi8(source) as u32; | |
| let ascii_prefix = high_mask.trailing_zeros() as usize; | |
| if ascii_prefix == 0 { | |
| break; | |
| } | |
| let low_nibbles = _mm256_and_si256(source, low_nibble_mask); | |
| let high_nibbles = _mm256_and_si256(_mm256_srli_epi16(source, 4), low_nibble_mask); | |
| let allowed_high_bits = _mm256_shuffle_epi8(low_table, low_nibbles); | |
| let observed_high_bits = _mm256_shuffle_epi8(high_table, high_nibbles); | |
| let collapse = _mm256_cmpeq_epi8( | |
| _mm256_and_si256(allowed_high_bits, observed_high_bits), | |
| observed_high_bits, | |
| ); | |
| let uppercase = _mm256_and_si256( | |
| _mm256_cmpgt_epi8(source, upper_lower_bound), | |
| _mm256_cmpgt_epi8(upper_upper_bound, source), | |
| ); | |
| let lowercased = _mm256_or_si256(source, _mm256_and_si256(uppercase, lowercase_bit)); | |
| let normalized_backtick = _mm256_blendv_epi8( | |
| lowercased, | |
| apostrophe, | |
| _mm256_cmpeq_epi8(source, backtick), | |
| ); | |
| unsafe { _mm256_storeu_si256(mapped.as_mut_ptr().cast(), normalized_backtick) }; | |
| emit_mixed_ascii_block( | |
| &mapped[..ascii_prefix], | |
| _mm256_movemask_epi8(collapse) as u32, | |
| source_base + consumed, | |
| normalized, | |
| offsets, | |
| pending_space, | |
| ); | |
| consumed += ascii_prefix; | |
| if ascii_prefix < 32 { | |
| break; | |
| } | |
| } | |
| consumed | |
| } | |
| fn normalize_text_into( | |
| text: &str, | |
| normalized: &mut Vec<u8>, | |
| offsets: &mut Vec<usize>, | |
| ) -> (bool, bool, usize) { | |
| normalize_text_into_with_mixed_avx2(text, normalized, offsets, true) | |
| } | |
| fn normalize_text_into_with_mixed_avx2( | |
| text: &str, | |
| normalized: &mut Vec<u8>, | |
| offsets: &mut Vec<usize>, | |
| allow_avx2: bool, | |
| ) -> (bool, bool, usize) { | |
| normalized.clear(); | |
| offsets.clear(); | |
| normalized.reserve(text.len()); | |
| offsets.reserve(text.len()); | |
| if text.is_ascii() { | |
| normalize_ascii_text_into(text, normalized, offsets); | |
| return (true, false, text.len()); | |
| } | |
| // Keep the whole-ASCII path above unchanged. Feature detection is cached | |
| // and selected once per mixed row, never without the runtime guard. | |
| let use_avx2 = allow_avx2 && mixed_ascii_avx2_available(); | |
| let _ = allow_avx2; | |
| let bytes = text.as_bytes(); | |
| let mut pending_space = false; | |
| let mut exceptional_ignorecase = false; | |
| let mut byte_cursor = 0usize; | |
| let mut source_len = 0usize; | |
| while byte_cursor < bytes.len() { | |
| if use_avx2 && bytes.len() - byte_cursor >= 32 && bytes[byte_cursor].is_ascii() { | |
| // SAFETY: use_avx2 includes runtime feature detection. The helper | |
| // bounds every load and emits only bytes before the first high bit. | |
| let consumed = unsafe { | |
| normalize_mixed_ascii_prefix_avx2_into( | |
| &bytes[byte_cursor..], source_len, normalized, offsets, &mut pending_space, | |
| ) | |
| }; | |
| byte_cursor += consumed; | |
| source_len += consumed; // Every consumed byte is one ASCII scalar. | |
| } | |
| // Consume raw ASCII tails once, without decoding their bytes as Unicode | |
| // scalars or searching the complete run before processing it. Original | |
| // code-point positions remain separate from UTF-8 byte positions. | |
| while byte_cursor < bytes.len() && bytes[byte_cursor].is_ascii() { | |
| let byte = normalize_ascii_byte(bytes[byte_cursor]); | |
| if is_python_ascii_space(byte) { | |
| pending_space = !normalized.is_empty(); | |
| } else { | |
| if pending_space { | |
| normalized.push(b' '); | |
| offsets.push(source_len); | |
| pending_space = false; | |
| } | |
| normalized.push(byte); | |
| offsets.push(source_len); | |
| } | |
| byte_cursor += 1; | |
| source_len += 1; | |
| } | |
| if byte_cursor == bytes.len() { | |
| break; | |
| } | |
| // The cursor starts at a character boundary and advances by one byte | |
| // for ASCII or by the complete decoded scalar's UTF-8 width. Keep the | |
| // frozen per-scalar mapping and repeated expansion offsets unchanged. | |
| let source_character = text[byte_cursor..] | |
| .chars() | |
| .next() | |
| .expect("remaining input contains a complete UTF-8 scalar"); | |
| exceptional_ignorecase |= matches!( | |
| source_character, | |
| '\u{0130}' | '\u{0131}' | '\u{017f}' | '\u{212a}' | |
| ); | |
| normalize_source_char( | |
| source_character, | |
| source_len, | |
| normalized, | |
| offsets, | |
| &mut pending_space, | |
| ); | |
| byte_cursor += source_character.len_utf8(); | |
| source_len += 1; | |
| } | |
| // The unchanged whole-row ASCII gate already rejected this input. | |
| (false, exceptional_ignorecase, source_len) | |
| } | |
| fn normalized_contains(haystack: &[u8], needle: &[u8]) -> bool { | |
| if needle.is_empty() || haystack.len() < needle.len() { | |
| return false; | |
| } | |
| let mut cursor = 0usize; | |
| while cursor + needle.len() <= haystack.len() { | |
| if let Some(relative) = haystack[cursor..] | |
| .windows(needle.len()) | |
| .position(|window| window == needle) | |
| { | |
| let start = cursor + relative; | |
| let end = start + needle.len(); | |
| let left_ok = start == 0 || !is_token_byte(haystack[start - 1]); | |
| let right_ok = end == haystack.len() || !is_token_byte(haystack[end]); | |
| if left_ok && right_ok { | |
| return true; | |
| } | |
| cursor = start + 1; | |
| } else { | |
| return false; | |
| } | |
| } | |
| false | |
| } | |
| fn normalized_context_window<'a>( | |
| normalized: &'a [u8], | |
| offsets: &[usize], | |
| source_len: usize, | |
| start: usize, | |
| end: usize, | |
| width: usize, | |
| ) -> &'a [u8] { | |
| debug_assert_eq!(normalized.len(), offsets.len()); | |
| debug_assert!(start <= end); | |
| debug_assert!(end <= source_len); | |
| let left = start.saturating_sub(width); | |
| let right = end.saturating_add(width).min(source_len); | |
| let normalized_left = offsets.partition_point(|&source_index| source_index < left); | |
| let normalized_right = offsets.partition_point(|&source_index| source_index < right); | |
| let mut window = &normalized[normalized_left..normalized_right]; | |
| // A full-row normalization can carry a collapsed separator from before the | |
| // source window to its first non-separator. Normalizing the source window in | |
| // isolation drops that leading separator, so remove it from the borrowed | |
| // slice as well. Trailing separators already map to the first source | |
| // character after the window and are excluded by normalized_right. | |
| if window.first() == Some(&b' ') { | |
| window = &window[1..]; | |
| } | |
| window | |
| } | |
| fn context_status( | |
| normalized: &[u8], | |
| offsets: &[usize], | |
| source_len: usize, | |
| start: usize, | |
| end: usize, | |
| rule: &ContextRule, | |
| ) -> u8 { | |
| // 0=satisfied, 1=explicit_collision_exclusion, 2=context_clause_not_satisfied | |
| let window = normalized_context_window( | |
| normalized, | |
| offsets, | |
| source_len, | |
| start, | |
| end, | |
| rule.window_chars, | |
| ); | |
| if rule | |
| .reject_terms | |
| .iter() | |
| .any(|term| normalized_contains(window, term)) | |
| { | |
| return 1; | |
| } | |
| if rule | |
| .accept_clauses | |
| .iter() | |
| .any(|clause| clause.iter().all(|term| normalized_contains(window, term))) | |
| { | |
| 0 | |
| } else { | |
| 2 | |
| } | |
| } | |
| fn py_ignorecase_ascii_literal_eq(pattern: char, observed: char) -> bool { | |
| let lower = pattern.to_ascii_lowercase(); | |
| if observed.is_ascii() { | |
| return observed.to_ascii_lowercase() == lower; | |
| } | |
| match lower { | |
| 'i' => matches!(observed, 'İ' | 'ı'), | |
| 's' => observed == 'ſ', | |
| 'k' => observed == 'K', | |
| _ => false, | |
| } | |
| } | |
| fn exact_matches(chars: &[char], spec: &ExactSpec) -> Vec<(usize, usize)> { | |
| let width = spec.value.len(); | |
| if width == 0 || width > chars.len() { | |
| return Vec::new(); | |
| } | |
| let mut output = Vec::new(); | |
| for start in 0..=chars.len() - width { | |
| let end = start + width; | |
| if start > 0 && is_exact_token_char(chars[start - 1], spec.mode) { | |
| continue; | |
| } | |
| if end < chars.len() && is_exact_token_char(chars[end], spec.mode) { | |
| continue; | |
| } | |
| let matched = spec.value.iter().enumerate().all(|(offset, expected)| { | |
| let observed = chars[start + offset]; | |
| match spec.mode { | |
| ExactMode::CaseSensitive => observed == *expected, | |
| ExactMode::PythonIgnoreCaseAsciiLiteral => { | |
| py_ignorecase_ascii_literal_eq(*expected, observed) | |
| } | |
| } | |
| }); | |
| if matched { | |
| output.push((start, end)); | |
| } | |
| } | |
| output | |
| } | |
| fn exact_boundaries_match(text: &str, start: usize, end: usize, mode: ExactMode) -> bool { | |
| let left_ok = start == 0 | |
| || !is_exact_token_char( | |
| text[..start] | |
| .chars() | |
| .next_back() | |
| .expect("nonempty exact left boundary"), | |
| mode, | |
| ); | |
| let right_ok = end == text.len() | |
| || !is_exact_token_char( | |
| text[end..] | |
| .chars() | |
| .next() | |
| .expect("nonempty exact right boundary"), | |
| mode, | |
| ); | |
| left_ok && right_ok | |
| } | |
| impl ExactAutomaton { | |
| fn append_matches(&self, text: &str, mode: ExactMode, output: &mut Vec<ExactByteMatch>) { | |
| for matched in self.automaton.find_overlapping_iter(text.as_bytes()) { | |
| let start = matched.start(); | |
| let end = matched.end(); | |
| if !exact_boundaries_match(text, start, end, mode) { | |
| continue; | |
| } | |
| for &spec_index in &self.pattern_specs[matched.pattern().as_usize()] { | |
| output.push(ExactByteMatch { | |
| spec_index, | |
| start, | |
| end, | |
| }); | |
| } | |
| } | |
| } | |
| } | |
| fn build_exact_automaton( | |
| specs: &[ExactSpec], | |
| mode: ExactMode, | |
| ascii_case_insensitive: bool, | |
| ) -> PyResult<Option<ExactAutomaton>> { | |
| let mut patterns = Vec::<String>::new(); | |
| let mut pattern_specs = Vec::<Vec<usize>>::new(); | |
| let mut pattern_ids = HashMap::<String, usize>::new(); | |
| for (spec_index, spec) in specs.iter().enumerate() { | |
| if spec.mode != mode { | |
| continue; | |
| } | |
| let value: String = spec.value.iter().collect(); | |
| let pattern_key = if ascii_case_insensitive { | |
| value.to_ascii_lowercase() | |
| } else { | |
| value.clone() | |
| }; | |
| if let Some(&pattern_id) = pattern_ids.get(&pattern_key) { | |
| pattern_specs[pattern_id].push(spec_index); | |
| } else { | |
| let pattern_id = patterns.len(); | |
| pattern_ids.insert(pattern_key, pattern_id); | |
| patterns.push(value); | |
| pattern_specs.push(vec![spec_index]); | |
| } | |
| } | |
| if patterns.is_empty() { | |
| return Ok(None); | |
| } | |
| let automaton = build_auto_automaton(&patterns, ascii_case_insensitive)?; | |
| Ok(Some(ExactAutomaton { | |
| automaton, | |
| pattern_specs, | |
| })) | |
| } | |
| impl ExactSpecs { | |
| fn new(specs: Vec<ExactSpec>) -> PyResult<Self> { | |
| let case_sensitive = build_exact_automaton(&specs, ExactMode::CaseSensitive, false)?; | |
| let ignorecase_ascii = | |
| build_exact_automaton(&specs, ExactMode::PythonIgnoreCaseAsciiLiteral, true)?; | |
| Ok(Self { | |
| specs, | |
| case_sensitive, | |
| ignorecase_ascii, | |
| }) | |
| } | |
| fn append_matching( | |
| &self, | |
| text: &str, | |
| byte_matches: &mut Vec<ExactByteMatch>, | |
| output: &mut Vec<(usize, usize, usize)>, | |
| exceptional_chars: &mut Vec<char>, | |
| boundaries: &mut Vec<usize>, | |
| text_is_ascii: bool, | |
| exceptional_ignorecase: bool, | |
| ) { | |
| byte_matches.clear(); | |
| output.clear(); | |
| exceptional_chars.clear(); | |
| boundaries.clear(); | |
| if let Some(automaton) = &self.case_sensitive { | |
| automaton.append_matches(text, ExactMode::CaseSensitive, byte_matches); | |
| } | |
| if !exceptional_ignorecase { | |
| if let Some(automaton) = &self.ignorecase_ascii { | |
| automaton.append_matches( | |
| text, | |
| ExactMode::PythonIgnoreCaseAsciiLiteral, | |
| byte_matches, | |
| ); | |
| } | |
| } | |
| output.reserve(byte_matches.len()); | |
| if text_is_ascii { | |
| output.extend( | |
| byte_matches | |
| .iter() | |
| .copied() | |
| .map(|matched| (matched.spec_index, matched.start, matched.end)), | |
| ); | |
| } else if !byte_matches.is_empty() { | |
| // Aho-Corasick reports UTF-8 byte offsets while the frozen Python | |
| // surface reports code-point offsets. Build this map only for the | |
| // uncommon non-ASCII row that actually produced an exact hit. | |
| boundaries.extend(text.char_indices().map(|(offset, _)| offset)); | |
| boundaries.push(text.len()); | |
| output.extend(byte_matches.iter().copied().map(|matched| { | |
| let start = boundaries | |
| .binary_search(&matched.start) | |
| .expect("exact match starts at a UTF-8 character boundary"); | |
| let end = boundaries | |
| .binary_search(&matched.end) | |
| .expect("exact match ends at a UTF-8 character boundary"); | |
| (matched.spec_index, start, end) | |
| })); | |
| } | |
| if exceptional_ignorecase { | |
| // Python re.IGNORECASE gives ASCII I/S/K four additional Unicode | |
| // equivalents (İ, ı, ſ, K). The common ASCII automaton cannot change | |
| // byte width while matching, so retain the exact scalar fallback | |
| // only for rows that contain one of those exceptional code points. | |
| exceptional_chars.extend(text.chars()); | |
| for (spec_index, spec) in self.specs.iter().enumerate() { | |
| if spec.mode != ExactMode::PythonIgnoreCaseAsciiLiteral { | |
| continue; | |
| } | |
| output.extend( | |
| exact_matches(exceptional_chars, spec) | |
| .into_iter() | |
| .map(|(start, end)| (spec_index, start, end)), | |
| ); | |
| } | |
| } | |
| } | |
| fn matching(&self, text: &str) -> Vec<(usize, usize, usize)> { | |
| let mut byte_matches = Vec::new(); | |
| let mut output = Vec::new(); | |
| let mut exceptional_chars = Vec::new(); | |
| let mut boundaries = Vec::new(); | |
| let mut normalized = Vec::new(); | |
| let mut offsets = Vec::new(); | |
| let (text_is_ascii, exceptional_ignorecase, _) = | |
| normalize_text_into(text, &mut normalized, &mut offsets); | |
| self.append_matching( | |
| text, | |
| &mut byte_matches, | |
| &mut output, | |
| &mut exceptional_chars, | |
| &mut boundaries, | |
| text_is_ascii, | |
| exceptional_ignorecase, | |
| ); | |
| output | |
| } | |
| } | |
| fn scan_one( | |
| automaton: &AhoCorasick, | |
| pattern_consumers: &[Vec<usize>], | |
| consumers: &[ConsumerSpec], | |
| exact_specs: &ExactSpecs, | |
| aliases: &[AliasSpec], | |
| lexical: &[LexicalSpec], | |
| contexts: &[ContextRule], | |
| conjunctions: &[ConjunctionSpec], | |
| text: &str, | |
| observed_epoch: i64, | |
| row_index: usize, | |
| scratch: &mut ScanScratch, | |
| ) -> Option<FactRow> { | |
| scratch.reset(conjunctions.len()); | |
| let (text_is_ascii, exceptional_ignorecase, source_len) = | |
| normalize_text_into(text, &mut scratch.normalized, &mut scratch.offsets); | |
| let mut native_candidate = false; | |
| if !scratch.normalized.is_empty() { | |
| for matched in automaton.find_overlapping_iter(&scratch.normalized) { | |
| let start = matched.start(); | |
| let end = matched.end(); | |
| let left_ok = start == 0 || !is_token_byte(scratch.normalized[start - 1]); | |
| let right_ok = | |
| end == scratch.normalized.len() || !is_token_byte(scratch.normalized[end]); | |
| if !left_ok || !right_ok { | |
| continue; | |
| } | |
| native_candidate = true; | |
| let original_start = scratch.offsets[start]; | |
| let original_end = scratch.offsets[end - 1] + 1; | |
| for &consumer_id in &pattern_consumers[matched.pattern().as_usize()] { | |
| let spec = &consumers[consumer_id]; | |
| match spec.kind { | |
| ALIAS => { | |
| scratch | |
| .alias_candidates | |
| .push((spec.a, original_start, original_end)); | |
| } | |
| LEXICAL => { | |
| scratch | |
| .lexical_candidates | |
| .push((spec.a, original_start, original_end)); | |
| } | |
| CONJUNCTION => { | |
| if conjunctions | |
| .get(spec.a) | |
| .is_some_and(|rule| rule.contains_phrase(spec.b, spec.c)) | |
| { | |
| scratch.conjunction_marks.mark(spec.a, spec.b); | |
| } | |
| } | |
| _ => unreachable!("validated consumer kind"), | |
| } | |
| } | |
| } | |
| } | |
| let mut exact_candidate = false; | |
| exact_specs.append_matching( | |
| text, | |
| &mut scratch.exact_byte_matches, | |
| &mut scratch.exact_matches, | |
| &mut scratch.chars, | |
| &mut scratch.exact_boundaries, | |
| text_is_ascii, | |
| exceptional_ignorecase, | |
| ); | |
| for &(spec_index, start, end) in &scratch.exact_matches { | |
| exact_candidate = true; | |
| let spec = &exact_specs.specs[spec_index]; | |
| match spec.kind { | |
| ALIAS => scratch.alias_candidates.push((spec.index, start, end)), | |
| LEXICAL => scratch.lexical_candidates.push((spec.index, start, end)), | |
| CONJUNCTION => { | |
| scratch.conjunction_marks.mark(spec.index, 0); | |
| } | |
| _ => unreachable!("validated exact kind"), | |
| } | |
| } | |
| if scratch.alias_candidates.is_empty() | |
| && scratch.lexical_candidates.is_empty() | |
| && scratch.conjunction_marks.is_empty() | |
| { | |
| return None; | |
| } | |
| let mut accepted_nonsemantic = false; | |
| let mut accepted_evidence = false; | |
| let mut unresolved_exposure = false; | |
| let mut unresolved_event = false; | |
| let mut supported_mask: u8 = 0; | |
| let mut observation_count = 0usize; | |
| // Group aliases by (canonical_id, start) without a per-row hash table. The | |
| // longest end in each group wins, while distinct aliases that tie remain | |
| // distinct observations. Exact duplicate candidates collapse here. | |
| scratch.alias_candidates.sort_unstable_by_key( | |
| |&(alias_index, start, end)| { | |
| (aliases[alias_index].canonical_group, start, end, alias_index) | |
| }, | |
| ); | |
| scratch.alias_candidates.dedup(); | |
| let mut group_start = 0usize; | |
| while group_start < scratch.alias_candidates.len() { | |
| let (first_alias, first_start, _) = scratch.alias_candidates[group_start]; | |
| let canonical_group = aliases[first_alias].canonical_group; | |
| let mut group_end = group_start + 1; | |
| let mut longest_end = scratch.alias_candidates[group_start].2; | |
| while group_end < scratch.alias_candidates.len() { | |
| let (alias_index, start, end) = scratch.alias_candidates[group_end]; | |
| if aliases[alias_index].canonical_group != canonical_group || start != first_start { | |
| break; | |
| } | |
| longest_end = longest_end.max(end); | |
| group_end += 1; | |
| } | |
| for candidate_index in group_start..group_end { | |
| let (alias_index, start, end) = scratch.alias_candidates[candidate_index]; | |
| if end < longest_end { | |
| continue; | |
| } | |
| let spec = &aliases[alias_index]; | |
| // The frozen compact contract counts a retained observation even | |
| // when its temporal or context checks later reject it. | |
| observation_count += 1; | |
| let active = observed_epoch >= spec.valid_from_epoch | |
| && spec | |
| .valid_to_epoch | |
| .map(|until| observed_epoch < until) | |
| .unwrap_or(true); | |
| if !active { | |
| continue; | |
| } | |
| let context_code = if let Some(rule_index) = spec.context_rule { | |
| context_status( | |
| &scratch.normalized, | |
| &scratch.offsets, | |
| source_len, | |
| start, | |
| end, | |
| &contexts[rule_index], | |
| ) | |
| } else { | |
| 0 | |
| }; | |
| if context_code != 0 { | |
| // Frozen exposure unresolved filter excludes explicit collision and | |
| // context-clause failures. Missing context cannot occur because the | |
| // validated config rejects unknown rules during construction. | |
| continue; | |
| } | |
| accepted_nonsemantic = true; | |
| if spec.family != FAMILY_NONE && (spec.entity_kind == 0 || spec.entity_kind == 1) { | |
| accepted_evidence = true; | |
| supported_mask |= 1u8 << (spec.family as u8); | |
| } else if spec.unresolved_tag { | |
| unresolved_exposure = true; | |
| } | |
| } | |
| group_start = group_end; | |
| } | |
| // Observation identity for lexical matches is (rule, start, end), so the | |
| // sorted sparse candidates replace the former observation HashSet exactly. | |
| scratch.lexical_candidates.sort_unstable(); | |
| scratch.lexical_candidates.dedup(); | |
| for candidate_index in 0..scratch.lexical_candidates.len() { | |
| let (lexical_index, start, end) = scratch.lexical_candidates[candidate_index]; | |
| let spec = &lexical[lexical_index]; | |
| observation_count += 1; | |
| let context_ok = if spec.context_terms.is_empty() { | |
| true | |
| } else { | |
| let window = normalized_context_window( | |
| &scratch.normalized, | |
| &scratch.offsets, | |
| source_len, | |
| start, | |
| end, | |
| spec.context_window_chars, | |
| ); | |
| spec.context_terms | |
| .iter() | |
| .any(|term| normalized_contains(window, term)) | |
| }; | |
| if spec.is_event { | |
| if spec.market_wide_authorized { | |
| accepted_evidence = true; | |
| supported_mask |= 1u8 << FAMILY_MARKET_WIDE; | |
| } else { | |
| if spec.unresolved_tag { | |
| unresolved_exposure = true; | |
| } | |
| if spec.has_event_family { | |
| unresolved_event = true; | |
| } | |
| } | |
| } else { | |
| if context_ok { | |
| accepted_nonsemantic = true; | |
| } | |
| if spec.market_wide_authorized && context_ok { | |
| accepted_evidence = true; | |
| supported_mask |= 1u8 << FAMILY_MARKET_WIDE; | |
| } else if spec.unresolved_tag { | |
| // Matches frozen generic lexical unresolved branch, including a | |
| // lexical context failure, whose evidence reason is the generic | |
| // identity-resolved-without-path code. | |
| unresolved_exposure = true; | |
| } | |
| } | |
| } | |
| for touched_index in 0..scratch.conjunction_marks.touched.len() { | |
| let rule_index = scratch.conjunction_marks.touched[touched_index]; | |
| if scratch.conjunction_marks.is_complete(rule_index) { | |
| // Only one conjunction observation exists per configured rule. | |
| observation_count += 1; | |
| accepted_evidence = true; | |
| supported_mask |= 1u8 << FAMILY_MARKET_WIDE; | |
| } | |
| } | |
| if observation_count == 0 { | |
| return None; | |
| } | |
| let article_status = if accepted_evidence || accepted_nonsemantic || unresolved_event { | |
| ARTICLE_ACCEPTED | |
| } else { | |
| ARTICLE_REJECTED | |
| }; | |
| let completed_elsewhere = supported_mask != 0; | |
| let exposure_status = |family: i8| -> u8 { | |
| if (supported_mask & (1u8 << (family as u8))) != 0 { | |
| EXP_SUPPORTED | |
| } else if family != FAMILY_MARKET_WIDE && !completed_elsewhere && unresolved_exposure { | |
| EXP_PLAUSIBLE | |
| } else { | |
| EXP_NOT_SUPPORTED | |
| } | |
| }; | |
| Some(( | |
| row_index, | |
| article_status, | |
| exposure_status(FAMILY_SP500), | |
| exposure_status(FAMILY_NASDAQ100), | |
| exposure_status(FAMILY_RUSSELL2000), | |
| exposure_status(FAMILY_MARKET_WIDE), | |
| observation_count, | |
| native_candidate, | |
| exact_candidate, | |
| )) | |
| } | |
| struct NativeCompactFactScanner { | |
| automaton: AhoCorasick, | |
| pattern_consumers: Vec<Vec<usize>>, | |
| consumers: Vec<ConsumerSpec>, | |
| exact_specs: ExactSpecs, | |
| aliases: Vec<AliasSpec>, | |
| lexical: Vec<LexicalSpec>, | |
| contexts: Vec<ContextRule>, | |
| conjunctions: Vec<ConjunctionSpec>, | |
| } | |
| impl NativeCompactFactScanner { | |
| fn new( | |
| patterns: Vec<String>, | |
| pattern_consumers: Vec<Vec<usize>>, | |
| consumers: Vec<(u8, usize, usize, usize)>, | |
| exact_specs: Vec<(String, String, u8, usize)>, | |
| alias_specs: Vec<(usize, i64, Option<i64>, i8, u8, bool, Option<usize>)>, | |
| lexical_specs: Vec<(usize, Vec<String>, bool, bool, bool, bool)>, | |
| context_rules: Vec<(usize, Vec<Vec<String>>, Vec<String>)>, | |
| conjunction_specs: Vec<(usize, usize, usize, usize)>, | |
| ) -> PyResult<Self> { | |
| if patterns.len() != pattern_consumers.len() { | |
| return Err(PyValueError::new_err("pattern_consumers length mismatch")); | |
| } | |
| let automaton = build_automaton(&patterns)?; | |
| let consumers = consumers | |
| .into_iter() | |
| .map(|(kind, a, b, c)| ConsumerSpec { kind, a, b, c }) | |
| .collect(); | |
| let mut parsed_exact = Vec::with_capacity(exact_specs.len()); | |
| for (value, mode, kind, index) in exact_specs { | |
| if value.is_empty() { | |
| return Err(PyValueError::new_err("exact value must be non-empty")); | |
| } | |
| let mode = match mode.as_str() { | |
| "case_sensitive" => ExactMode::CaseSensitive, | |
| "python_ignorecase_ascii_literal" => { | |
| if !value.is_ascii() { | |
| return Err(PyValueError::new_err("ignorecase literal must be ASCII")); | |
| } | |
| ExactMode::PythonIgnoreCaseAsciiLiteral | |
| } | |
| _ => return Err(PyValueError::new_err(format!("unknown exact mode: {mode}"))), | |
| }; | |
| parsed_exact.push(ExactSpec { | |
| value: value.chars().collect(), | |
| mode, | |
| kind, | |
| index, | |
| }); | |
| } | |
| let aliases = alias_specs | |
| .into_iter() | |
| .map( | |
| |( | |
| canonical_group, | |
| valid_from_epoch, | |
| valid_to_epoch, | |
| family, | |
| entity_kind, | |
| unresolved_tag, | |
| context_rule, | |
| )| AliasSpec { | |
| canonical_group, | |
| valid_from_epoch, | |
| valid_to_epoch, | |
| family, | |
| entity_kind, | |
| unresolved_tag, | |
| context_rule, | |
| }, | |
| ) | |
| .collect(); | |
| let lexical = lexical_specs | |
| .into_iter() | |
| .map( | |
| |( | |
| context_window_chars, | |
| terms, | |
| is_event, | |
| market_wide_authorized, | |
| unresolved_tag, | |
| has_event_family, | |
| )| LexicalSpec { | |
| context_window_chars, | |
| context_terms: terms.into_iter().map(String::into_bytes).collect(), | |
| is_event, | |
| market_wide_authorized, | |
| unresolved_tag, | |
| has_event_family, | |
| }, | |
| ) | |
| .collect(); | |
| let contexts = context_rules | |
| .into_iter() | |
| .map(|(window_chars, clauses, rejects)| ContextRule { | |
| window_chars, | |
| accept_clauses: clauses | |
| .into_iter() | |
| .map(|clause| clause.into_iter().map(String::into_bytes).collect()) | |
| .collect(), | |
| reject_terms: rejects.into_iter().map(String::into_bytes).collect(), | |
| }) | |
| .collect(); | |
| let conjunctions = conjunction_specs | |
| .into_iter() | |
| .map( | |
| |( | |
| actor_phrase_count, | |
| action_phrase_count, | |
| object_phrase_count, | |
| scope_phrase_count, | |
| )| ConjunctionSpec { | |
| actor_phrase_count, | |
| action_phrase_count, | |
| object_phrase_count, | |
| scope_phrase_count, | |
| }, | |
| ) | |
| .collect(); | |
| let exact_specs = ExactSpecs::new(parsed_exact)?; | |
| Ok(Self { | |
| automaton, | |
| pattern_consumers, | |
| consumers, | |
| exact_specs, | |
| aliases, | |
| lexical, | |
| contexts, | |
| conjunctions, | |
| }) | |
| } | |
| fn screen_facts_batch( | |
| &self, | |
| py: Python<'_>, | |
| texts: Vec<String>, | |
| epochs: Vec<i64>, | |
| workers: usize, | |
| ) -> PyResult<Vec<FactRow>> { | |
| if texts.len() != epochs.len() { | |
| return Err(PyValueError::new_err("texts/epochs length mismatch")); | |
| } | |
| if workers == 0 { | |
| return Err(PyValueError::new_err("workers must be >= 1")); | |
| } | |
| let pool = ThreadPoolBuilder::new() | |
| .num_threads(workers) | |
| .thread_name(|index| format!("news-relevance-worker-{index}")) | |
| .build() | |
| .map_err(|error| PyValueError::new_err(error.to_string()))?; | |
| let automaton = &self.automaton; | |
| let pattern_consumers = &self.pattern_consumers; | |
| let consumers = &self.consumers; | |
| let exact_specs = &self.exact_specs; | |
| let aliases = &self.aliases; | |
| let lexical = &self.lexical; | |
| let contexts = &self.contexts; | |
| let conjunctions = &self.conjunctions; | |
| Ok(py.detach(move || { | |
| pool.install(|| { | |
| texts | |
| .par_iter() | |
| .zip(epochs.par_iter()) | |
| .enumerate() | |
| .map_init(ScanScratch::default, |scratch, (index, (text, epoch))| { | |
| scan_one( | |
| automaton, | |
| pattern_consumers, | |
| consumers, | |
| exact_specs, | |
| aliases, | |
| lexical, | |
| contexts, | |
| conjunctions, | |
| text, | |
| *epoch, | |
| index, | |
| scratch, | |
| ) | |
| }) | |
| .filter_map(|row| row) | |
| .collect() | |
| }) | |
| })) | |
| } | |
| } | |
| mod matcher_tests { | |
| use super::*; | |
| fn spec(value: &str, mode: ExactMode, index: usize) -> ExactSpec { | |
| ExactSpec { | |
| value: value.chars().collect(), | |
| mode, | |
| kind: ALIAS, | |
| index, | |
| } | |
| } | |
| fn synthetic_patterns(count: usize) -> Vec<String> { | |
| (0..count) | |
| .map(|index| format!("synthetic pattern {index:04}")) | |
| .collect() | |
| } | |
| fn overlapping_matches(automaton: &AhoCorasick, text: &str) -> Vec<(usize, usize, usize)> { | |
| let mut matches: Vec<_> = automaton | |
| .find_overlapping_iter(text.as_bytes()) | |
| .map(|matched| { | |
| ( | |
| matched.pattern().as_usize(), | |
| matched.start(), | |
| matched.end(), | |
| ) | |
| }) | |
| .collect(); | |
| matches.sort_unstable(); | |
| matches | |
| } | |
| fn default_sized_normalized_taxonomy_selects_bounded_dfa() { | |
| // The maintained taxonomy currently has 226 normalized patterns, so | |
| // this sits above aho-corasick's automatic DFA threshold while still | |
| // remaining comfortably inside RANGE's explicit safety bounds. | |
| let patterns = synthetic_patterns(226); | |
| assert!(normalized_dfa_eligible(&patterns)); | |
| let automaton = build_automaton(&patterns).expect("bounded DFA builds"); | |
| assert_eq!(automaton.kind(), AhoCorasickKind::DFA); | |
| assert!(automaton.memory_usage() <= NORMALIZED_DFA_MAX_HEAP_BYTES); | |
| } | |
| fn large_custom_taxonomy_falls_back_to_ordinary_auto_selection() { | |
| let patterns = synthetic_patterns(NORMALIZED_DFA_MAX_PATTERNS + 1); | |
| assert!(!normalized_dfa_eligible(&patterns)); | |
| let selected = build_automaton(&patterns).expect("fallback automaton builds"); | |
| let automatic = build_auto_automaton(&patterns, false).expect("auto automaton builds"); | |
| assert_eq!(selected.kind(), automatic.kind()); | |
| assert_ne!(selected.kind(), AhoCorasickKind::DFA); | |
| } | |
| fn bounded_dfa_preserves_standard_overlapping_matches() { | |
| let mut patterns = synthetic_patterns(101); | |
| patterns.extend([ | |
| "equity market".to_string(), | |
| "equity market selloff".to_string(), | |
| "market selloff".to_string(), | |
| "capital rule".to_string(), | |
| "rule".to_string(), | |
| ]); | |
| let text = "a capital rule preceded an equity market selloff"; | |
| let selected = build_automaton(&patterns).expect("bounded DFA builds"); | |
| let automatic = build_auto_automaton(&patterns, false).expect("auto automaton builds"); | |
| assert_eq!(selected.kind(), AhoCorasickKind::DFA); | |
| assert_ne!(automatic.kind(), AhoCorasickKind::DFA); | |
| assert_eq!(selected.match_kind(), MatchKind::Standard); | |
| assert_eq!( | |
| overlapping_matches(&selected, text), | |
| overlapping_matches(&automatic, text) | |
| ); | |
| } | |
| fn exact_automata_equal_scalar_reference_across_boundaries_and_unicode() { | |
| let specs = vec![ | |
| spec("UPRO", ExactMode::CaseSensitive, 0), | |
| spec("QQQ", ExactMode::PythonIgnoreCaseAsciiLiteral, 1), | |
| spec("iShares", ExactMode::PythonIgnoreCaseAsciiLiteral, 2), | |
| spec("PowerShares", ExactMode::PythonIgnoreCaseAsciiLiteral, 3), | |
| // Duplicate literals must still fan out to every configured consumer. | |
| spec("SEC", ExactMode::CaseSensitive, 4), | |
| spec("SEC", ExactMode::CaseSensitive, 5), | |
| ]; | |
| let matcher = ExactSpecs::new(specs).expect("valid exact automata"); | |
| let texts = [ | |
| "UPRO upro QQQ qqq xQQQ QQQx SEC", | |
| "é UPRO αQQQ αSEC", | |
| "İShares ıSHARES iShares Powerſhares", | |
| "İQQQ QQQİ ſQQQ QQQK", | |
| "no exact literals here", | |
| ]; | |
| for text in texts { | |
| let chars: Vec<char> = text.chars().collect(); | |
| let mut expected = Vec::new(); | |
| for (spec_index, spec) in matcher.specs.iter().enumerate() { | |
| expected.extend( | |
| exact_matches(&chars, spec) | |
| .into_iter() | |
| .map(|(start, end)| (spec_index, start, end)), | |
| ); | |
| } | |
| expected.sort_unstable(); | |
| let mut observed = matcher.matching(text); | |
| observed.sort_unstable(); | |
| assert_eq!(observed, expected, "text={text:?}"); | |
| assert_eq!(!observed.is_empty(), !expected.is_empty()); | |
| } | |
| } | |
| fn python_ignorecase_exceptions_remain_token_boundaries() { | |
| let matcher = ExactSpecs::new(vec![spec( | |
| "QQQ", | |
| ExactMode::PythonIgnoreCaseAsciiLiteral, | |
| 0, | |
| )]) | |
| .expect("valid exact automaton"); | |
| for text in ["İQQQ", "QQQı", "ſQQQ", "QQQK"] { | |
| assert!(matcher.matching(text).is_empty(), "text={text:?}"); | |
| } | |
| } | |
| fn direct_text_normalization_matches_character_slice_reference() { | |
| let texts = [ | |
| "ASCII -- punctuation\tand spaces", | |
| "Café — ‘quoted’ ／ fullwidth", | |
| "Straße İ ı ſ K fi", | |
| "\u{001c}leading\u{00a0}and\u{2029}trailing\n", | |
| "", | |
| ]; | |
| for text in texts { | |
| let chars: Vec<char> = text.chars().collect(); | |
| let mut expected = Vec::new(); | |
| let mut expected_offsets = Vec::new(); | |
| normalize_chars_into(&chars, 0, &mut expected, &mut expected_offsets); | |
| let mut observed = Vec::new(); | |
| let mut observed_offsets = Vec::new(); | |
| let (all_ascii, exceptional_ignorecase, source_len) = | |
| normalize_text_into(text, &mut observed, &mut observed_offsets); | |
| assert_eq!(observed, expected, "text={text:?}"); | |
| assert_eq!(observed_offsets, expected_offsets, "text={text:?}"); | |
| assert_eq!(source_len, chars.len(), "text={text:?}"); | |
| assert_eq!( | |
| all_ascii, | |
| chars.iter().all(|character| character.is_ascii()) | |
| ); | |
| assert_eq!( | |
| exceptional_ignorecase, | |
| chars.iter().any(|character| matches!( | |
| character, | |
| '\u{0130}' | '\u{0131}' | '\u{017f}' | '\u{212a}' | |
| )), | |
| "text={text:?}" | |
| ); | |
| } | |
| } | |
| // Reuse all reference/candidate buffers across exhaustive synthetic cases. | |
| // This keeps auxiliary memory bounded by the longest individual fixture. | |
| fn normalization_reference_checker() -> impl FnMut(&str) { | |
| let mut chars = Vec::new(); | |
| let mut expected = Vec::new(); | |
| let mut expected_offsets = Vec::new(); | |
| let mut observed = Vec::new(); | |
| let mut observed_offsets = Vec::new(); | |
| move |text: &str| { | |
| chars.clear(); | |
| chars.extend(text.chars()); | |
| normalize_chars_into(&chars, 0, &mut expected, &mut expected_offsets); | |
| // The existing exhaustive scalar and boundary fixtures explicitly | |
| // exercise both forced scalar and runtime-guarded vector dispatch. | |
| // On non-AVX2 machines both are portable scalar; vector status must | |
| // be reported as unexercised, not counted as a passing vector gate. | |
| for allow_avx2 in [false, true] { | |
| let flags = normalize_text_into_with_mixed_avx2( | |
| text, &mut observed, &mut observed_offsets, allow_avx2, | |
| ); | |
| assert_eq!(observed, expected, "allow_avx2={allow_avx2} text={text:?}"); | |
| assert_eq!(observed_offsets, expected_offsets, | |
| "allow_avx2={allow_avx2} text={text:?}"); | |
| assert_eq!( | |
| flags, | |
| ( | |
| text.is_ascii(), | |
| chars.iter().any(|character| matches!( | |
| character, | |
| '\u{0130}' | '\u{0131}' | '\u{017f}' | '\u{212a}' | |
| )), | |
| chars.len(), | |
| ), | |
| "allow_avx2={allow_avx2} text={text:?}", | |
| ); | |
| } | |
| } | |
| } | |
| fn mixed_ascii_runs_match_reference_for_every_valid_scalar() { | |
| let mut check = normalization_reference_checker(); | |
| // Reuse one bounded buffer. Each non-ASCII scalar has a complete | |
| // vector block on both sides; ASCII scalars occur inside a vector block. | |
| // NBSP keeps every scalar fixture on the mixed-row branch. Both forced | |
| // scalar and guarded vector modes run in the shared reference checker. | |
| let prefix = "A \t-B`/ ".repeat(4); | |
| let suffix = " /C D`B ".repeat(4); | |
| assert_eq!(prefix.len(), 32); | |
| assert_eq!(suffix.len(), 32); | |
| let mut text = String::with_capacity(prefix.len() + suffix.len() + 8); | |
| // One fixture per valid scalar, not a scalar/run-length Cartesian product. | |
| for codepoint in 0..=0x10ffff { | |
| let Some(character) = char::from_u32(codepoint) else { | |
| continue; | |
| }; | |
| text.clear(); | |
| text.push_str(&prefix); | |
| text.push(character); | |
| text.push_str(&suffix); | |
| text.push_str("\u{00a0}D"); | |
| check(&text); | |
| } | |
| } | |
| fn mixed_ascii_runs_preserve_all_ascii_bytes_at_unicode_boundaries() { | |
| let mut check = normalization_reference_checker(); | |
| let mut text = String::with_capacity(160); | |
| let unicode = [ | |
| '\u{0080}', 'é', '\u{0301}', '—', '\u{00a0}', '…', '\u{2029}', | |
| 'fi', 'İ', 'ı', 'ſ', 'K', '界', '🦀', '\u{fdfa}', '\u{10ffff}', | |
| ]; | |
| for run_length in [0, 1, 15, 16, 31, 32, 33, 63, 64, 65, 127, 128, 129] { | |
| for byte in 0u8..=0x7f { | |
| for character in unicode { | |
| text.clear(); | |
| text.extend(std::iter::repeat('A').take(run_length)); | |
| text.push(byte as char); | |
| text.push(character); | |
| text.push(byte as char); | |
| text.push_str(" \tB/"); | |
| check(&text); | |
| // Also cover a Unicode scalar first and last, with the | |
| // same ASCII byte on both sides of a variable-length run. | |
| text.clear(); | |
| text.push(character); | |
| text.push(byte as char); | |
| text.extend(std::iter::repeat('Z').take(run_length)); | |
| text.push(byte as char); | |
| text.push(character); | |
| check(&text); | |
| } | |
| } | |
| } | |
| } | |
| fn mixed_ascii_runs_keep_pending_space_and_expansion_source_offsets() { | |
| let mut normalized = Vec::new(); | |
| let mut offsets = Vec::new(); | |
| assert_eq!( | |
| normalize_text_into("\u{00a0}\u{001c}É — \tB", &mut normalized, &mut offsets), | |
| (false, false, 8), | |
| ); | |
| assert_eq!(normalized, "é b".as_bytes()); | |
| assert_eq!(offsets, [2, 2, 7, 7]); | |
| assert_eq!( | |
| normalize_text_into("A\u{00a0}fi\u{001c}B", &mut normalized, &mut offsets), | |
| (false, false, 5), | |
| ); | |
| assert_eq!(normalized, b"a fi b"); | |
| assert_eq!(offsets, [0, 2, 2, 2, 4, 4]); | |
| let mut check = normalization_reference_checker(); | |
| for text in [ | |
| "éα界🦀", "İIıſK", "\u{00a0}\u{2029}—", "A\u{0301}B", | |
| "\u{0301}A\u{0301}", "A—\u{00a0}\u{001c}B", "A\u{fdfa}B", | |
| ] { | |
| check(text); | |
| } | |
| } | |
| fn assert_ascii_normalization_matches_character_reference(bytes: &[u8]) { | |
| let text = std::str::from_utf8(bytes).expect("ASCII bytes are valid UTF-8"); | |
| let chars: Vec<char> = text.chars().collect(); | |
| let mut expected = Vec::new(); | |
| let mut expected_offsets = Vec::new(); | |
| normalize_chars_into(&chars, 0, &mut expected, &mut expected_offsets); | |
| let mut observed = Vec::new(); | |
| let mut observed_offsets = Vec::new(); | |
| let flags = normalize_text_into(text, &mut observed, &mut observed_offsets); | |
| assert_eq!(flags, (true, false, bytes.len()), "bytes={bytes:?}"); | |
| assert_eq!(observed, expected, "bytes={bytes:?}"); | |
| assert_eq!(observed_offsets, expected_offsets, "bytes={bytes:?}"); | |
| } | |
| fn ascii_byte_path_matches_character_reference_exhaustively() { | |
| assert_ascii_normalization_matches_character_reference(b""); | |
| for first in 0u8..=0x7f { | |
| assert_ascii_normalization_matches_character_reference(&[first]); | |
| assert_ascii_normalization_matches_character_reference(&[b'x', first, b'y']); | |
| for second in 0u8..=0x7f { | |
| assert_ascii_normalization_matches_character_reference(&[first, second]); | |
| assert_ascii_normalization_matches_character_reference(&[ | |
| b'x', first, second, b'y', | |
| ]); | |
| } | |
| } | |
| } | |
| fn ascii_byte_path_preserves_collapse_and_control_semantics() { | |
| let tricky = [ | |
| b' ', b'-', b'\t', b'A', b'`', b'B', b'.', b'/', b' ', 0x00, b'\\', 0x7f, b' ', b'\n', | |
| ]; | |
| assert_ascii_normalization_matches_character_reference(&tricky); | |
| let mut normalized = Vec::new(); | |
| let mut offsets = Vec::new(); | |
| let text = std::str::from_utf8(&tricky).expect("ASCII bytes are valid UTF-8"); | |
| assert_eq!( | |
| normalize_text_into(text, &mut normalized, &mut offsets), | |
| (true, false, tricky.len()) | |
| ); | |
| assert_eq!(normalized, b"a'b \0\\\x7f"); | |
| assert_eq!(offsets, [3, 4, 5, 9, 9, 10, 11]); | |
| let python_ascii_whitespace = [ | |
| b' ', b'\t', b'\n', 0x0b, 0x0c, b'\r', 0x1c, 0x1d, 0x1e, 0x1f, | |
| ]; | |
| let separators = [ | |
| b'-', b'.', b',', b':', b';', b'(', b')', b'[', b']', b'{', b'}', b'/', | |
| ]; | |
| let mut runs = vec![b'A']; | |
| runs.extend_from_slice(&python_ascii_whitespace); | |
| runs.extend_from_slice(&separators); | |
| runs.push(b'Z'); | |
| assert_ascii_normalization_matches_character_reference(&runs); | |
| normalize_text_into( | |
| std::str::from_utf8(&runs).expect("ASCII bytes are valid UTF-8"), | |
| &mut normalized, | |
| &mut offsets, | |
| ); | |
| assert_eq!(normalized, b"a z"); | |
| assert_eq!(offsets, [0, runs.len() - 1, runs.len() - 1]); | |
| } | |
| fn borrowed_context_windows_match_isolated_normalization_exhaustively() { | |
| let texts = [ | |
| "", | |
| "a", | |
| " --Federal\tReserve...cut rates ", | |
| "prefix---GDP\u{00a0}growth／suffix", | |
| "Café — ‘SEC’ ／ filing", | |
| "İ ı ſ K fi — SPY ETF", | |
| "x\u{001c}\u{001d}market\u{2029}structure\n", | |
| ]; | |
| for text in texts { | |
| let chars: Vec<char> = text.chars().collect(); | |
| let mut normalized = Vec::new(); | |
| let mut offsets = Vec::new(); | |
| let (_, _, source_len) = normalize_text_into(text, &mut normalized, &mut offsets); | |
| assert_eq!(source_len, chars.len()); | |
| for start in 0..=source_len { | |
| for end in start..=source_len { | |
| for width in [0, 1, 2, 4, source_len.saturating_add(2)] { | |
| let left = start.saturating_sub(width); | |
| let right = end.saturating_add(width).min(source_len); | |
| let mut expected = Vec::new(); | |
| let mut expected_offsets = Vec::new(); | |
| normalize_chars_into( | |
| &chars[left..right], | |
| left, | |
| &mut expected, | |
| &mut expected_offsets, | |
| ); | |
| let observed = normalized_context_window( | |
| &normalized, | |
| &offsets, | |
| source_len, | |
| start, | |
| end, | |
| width, | |
| ); | |
| assert_eq!( | |
| observed, expected, | |
| "text={text:?}, start={start}, end={end}, width={width}" | |
| ); | |
| } | |
| } | |
| } | |
| } | |
| } | |
| fn borrowed_context_windows_preserve_phrase_boundaries() { | |
| let text = "prefederal -- Federal Reserve; GDP-growth; SECond SEC filing; SPY ETF"; | |
| let chars: Vec<char> = text.chars().collect(); | |
| let mut normalized = Vec::new(); | |
| let mut offsets = Vec::new(); | |
| let (_, _, source_len) = normalize_text_into(text, &mut normalized, &mut offsets); | |
| let terms: [&[u8]; 7] = [ | |
| b"fed", | |
| b"federal reserve", | |
| b"gdp", | |
| b"growth", | |
| b"sec", | |
| b"spy", | |
| b"etf", | |
| ]; | |
| for start in 0..=source_len { | |
| for end in start..=source_len { | |
| for width in [0, 1, 8, source_len.saturating_add(1)] { | |
| let left = start.saturating_sub(width); | |
| let right = end.saturating_add(width).min(source_len); | |
| let mut expected = Vec::new(); | |
| let mut expected_offsets = Vec::new(); | |
| normalize_chars_into( | |
| &chars[left..right], | |
| left, | |
| &mut expected, | |
| &mut expected_offsets, | |
| ); | |
| let observed = normalized_context_window( | |
| &normalized, | |
| &offsets, | |
| source_len, | |
| start, | |
| end, | |
| width, | |
| ); | |
| for term in terms { | |
| assert_eq!( | |
| normalized_contains(observed, term), | |
| normalized_contains(&expected, term), | |
| "term={term:?}, start={start}, end={end}, width={width}" | |
| ); | |
| } | |
| } | |
| } | |
| } | |
| } | |
| fn custom_spy_context_keeps_reject_precedence() { | |
| let automaton = | |
| build_automaton(&["never matches".to_string()]).expect("valid normalized automaton"); | |
| let exact_specs = ExactSpecs::new(vec![ExactSpec { | |
| value: "SPY".chars().collect(), | |
| mode: ExactMode::CaseSensitive, | |
| kind: ALIAS, | |
| index: 0, | |
| }]) | |
| .expect("valid exact alias"); | |
| let aliases = vec![AliasSpec { | |
| canonical_group: 0, | |
| context_rule: Some(0), | |
| valid_from_epoch: 0, | |
| valid_to_epoch: None, | |
| family: FAMILY_SP500, | |
| entity_kind: 0, | |
| unresolved_tag: false, | |
| }]; | |
| let contexts = vec![ContextRule { | |
| window_chars: 128, | |
| accept_clauses: vec![ | |
| vec![b"etf".to_vec()], | |
| vec![b"fund".to_vec(), b"s&p 500".to_vec()], | |
| ], | |
| reject_terms: vec![b"spy plane".to_vec(), b"espionage".to_vec()], | |
| }]; | |
| let mut scratch = ScanScratch::default(); | |
| let scan = |text: &str, row_index: usize, scratch: &mut ScanScratch| { | |
| scan_one( | |
| &automaton, | |
| &[Vec::new()], | |
| &[], | |
| &exact_specs, | |
| &aliases, | |
| &[], | |
| &contexts, | |
| &[], | |
| text, | |
| 1, | |
| row_index, | |
| scratch, | |
| ) | |
| .expect("SPY remains an observed alias") | |
| }; | |
| let accepted = scan("SPY — ETF shares rose", 0, &mut scratch); | |
| assert_eq!(accepted.1, ARTICLE_ACCEPTED); | |
| assert_eq!(accepted.2, EXP_SUPPORTED); | |
| let rejected = scan("SPY ETF appeared beside a spy plane", 1, &mut scratch); | |
| assert_eq!(rejected.1, ARTICLE_REJECTED); | |
| assert_eq!(rejected.2, EXP_NOT_SUPPORTED); | |
| let missing = scan("SPY reported quarterly earnings", 2, &mut scratch); | |
| assert_eq!(missing.1, ARTICLE_REJECTED); | |
| assert_eq!(missing.2, EXP_NOT_SUPPORTED); | |
| } | |
| fn representative_fed_gdp_and_sec_contexts_keep_frozen_behavior() { | |
| let automaton = | |
| build_automaton(&["federal reserve".to_string()]).expect("valid normalized automaton"); | |
| let pattern_consumers = vec![vec![0]]; | |
| let consumers = vec![ConsumerSpec { | |
| kind: LEXICAL, | |
| a: 0, | |
| b: 0, | |
| c: 0, | |
| }]; | |
| let exact_specs = ExactSpecs::new(vec![ | |
| ExactSpec { | |
| value: "GDP".chars().collect(), | |
| mode: ExactMode::CaseSensitive, | |
| kind: LEXICAL, | |
| index: 1, | |
| }, | |
| ExactSpec { | |
| value: "SEC".chars().collect(), | |
| mode: ExactMode::CaseSensitive, | |
| kind: LEXICAL, | |
| index: 2, | |
| }, | |
| ]) | |
| .expect("valid exact lexical matchers"); | |
| let lexical = vec![ | |
| LexicalSpec { | |
| context_window_chars: 0, | |
| context_terms: Vec::new(), | |
| is_event: false, | |
| market_wide_authorized: true, | |
| unresolved_tag: false, | |
| has_event_family: false, | |
| }, | |
| LexicalSpec { | |
| context_window_chars: 64, | |
| context_terms: vec![b"growth".to_vec(), b"economy".to_vec()], | |
| is_event: false, | |
| market_wide_authorized: true, | |
| unresolved_tag: false, | |
| has_event_family: false, | |
| }, | |
| LexicalSpec { | |
| context_window_chars: 96, | |
| context_terms: vec![b"commission".to_vec(), b"filing".to_vec()], | |
| is_event: false, | |
| market_wide_authorized: false, | |
| unresolved_tag: false, | |
| has_event_family: false, | |
| }, | |
| ]; | |
| let mut scratch = ScanScratch::default(); | |
| let scan = |text: &str, row_index: usize, scratch: &mut ScanScratch| { | |
| scan_one( | |
| &automaton, | |
| &pattern_consumers, | |
| &consumers, | |
| &exact_specs, | |
| &[], | |
| &lexical, | |
| &[], | |
| &[], | |
| text, | |
| 1, | |
| row_index, | |
| scratch, | |
| ) | |
| }; | |
| let fed = scan("The Federal Reserve cut rates", 0, &mut scratch) | |
| .expect("Federal Reserve is relevant"); | |
| assert_eq!(fed.1, ARTICLE_ACCEPTED); | |
| assert_eq!(fed.5, EXP_SUPPORTED); | |
| let gdp = scan("Economic growth lifted GDP", 1, &mut scratch) | |
| .expect("contextual GDP is observed"); | |
| assert_eq!(gdp.1, ARTICLE_ACCEPTED); | |
| assert_eq!(gdp.5, EXP_SUPPORTED); | |
| let gdp_without_context = scan("The GDP widget was renamed", 2, &mut scratch) | |
| .expect("GDP remains an observation when context fails"); | |
| assert_eq!(gdp_without_context.1, ARTICLE_REJECTED); | |
| assert_eq!(gdp_without_context.5, EXP_NOT_SUPPORTED); | |
| let sec = scan("The SEC filing was published", 3, &mut scratch) | |
| .expect("contextual SEC is observed"); | |
| assert_eq!(sec.1, ARTICLE_ACCEPTED); | |
| assert_eq!(sec.5, EXP_NOT_SUPPORTED); | |
| assert!(scan("SECond place", 4, &mut scratch).is_none()); | |
| } | |
| fn reused_scan_scratch_does_not_leak_candidates_between_rows() { | |
| let automaton = build_automaton(&["alpha".to_string()]).expect("valid automaton"); | |
| let pattern_consumers = vec![vec![0]]; | |
| let consumers = vec![ConsumerSpec { | |
| kind: ALIAS, | |
| a: 0, | |
| b: 0, | |
| c: 0, | |
| }]; | |
| let exact_specs = ExactSpecs::new(Vec::new()).expect("empty exact specs are valid"); | |
| let aliases = vec![AliasSpec { | |
| canonical_group: 0, | |
| context_rule: None, | |
| valid_from_epoch: 0, | |
| valid_to_epoch: None, | |
| family: FAMILY_SP500, | |
| entity_kind: 0, | |
| unresolved_tag: false, | |
| }]; | |
| let mut scratch = ScanScratch::default(); | |
| let scan = |text: &str, row_index: usize, scratch: &mut ScanScratch| { | |
| scan_one( | |
| &automaton, | |
| &pattern_consumers, | |
| &consumers, | |
| &exact_specs, | |
| &aliases, | |
| &[], | |
| &[], | |
| &[], | |
| text, | |
| 1, | |
| row_index, | |
| scratch, | |
| ) | |
| }; | |
| let first = scan("Alpha rises", 0, &mut scratch).expect("first row matches"); | |
| assert_eq!(first.0, 0); | |
| assert_eq!(first.6, 1); | |
| assert!(scan("nothing relevant", 1, &mut scratch).is_none()); | |
| let third = scan("alpha again", 2, &mut scratch).expect("third row matches"); | |
| assert_eq!(third.0, 2); | |
| assert_eq!(third.6, 1); | |
| } | |
| fn conjunction_marks_resize_reset_and_survive_generation_wrap() { | |
| let mut marks = ConjunctionMarks::default(); | |
| marks.begin_row(130); | |
| for field in 0..4 { | |
| marks.mark(129, field); | |
| } | |
| assert_eq!(marks.touched, [129]); | |
| assert!(marks.is_complete(129)); | |
| marks.begin_row(130); | |
| assert!(marks.is_empty()); | |
| assert!(!marks.is_complete(129)); | |
| marks.generation = u32::MAX; | |
| marks.begin_row(130); | |
| assert_eq!(marks.generation, 1); | |
| assert!(marks.stamps.iter().all(|stamp| *stamp == 0)); | |
| assert!(!marks.is_complete(129)); | |
| } | |
| fn sorted_alias_candidates_preserve_longest_ties_and_frozen_counting() { | |
| let automaton = | |
| build_automaton(&["alpha".to_string(), "alpha beta".to_string()]) | |
| .expect("valid automaton"); | |
| let pattern_consumers = vec![vec![0], vec![1, 1, 2]]; | |
| let consumers = vec![ | |
| ConsumerSpec { | |
| kind: ALIAS, | |
| a: 0, | |
| b: 0, | |
| c: 0, | |
| }, | |
| ConsumerSpec { | |
| kind: ALIAS, | |
| a: 1, | |
| b: 0, | |
| c: 0, | |
| }, | |
| ConsumerSpec { | |
| kind: ALIAS, | |
| a: 2, | |
| b: 0, | |
| c: 0, | |
| }, | |
| ]; | |
| let aliases = vec![ | |
| AliasSpec { | |
| canonical_group: 0, | |
| context_rule: None, | |
| valid_from_epoch: 0, | |
| valid_to_epoch: None, | |
| family: FAMILY_SP500, | |
| entity_kind: 0, | |
| unresolved_tag: false, | |
| }, | |
| AliasSpec { | |
| canonical_group: 0, | |
| context_rule: None, | |
| valid_from_epoch: 0, | |
| valid_to_epoch: None, | |
| family: FAMILY_SP500, | |
| entity_kind: 0, | |
| unresolved_tag: false, | |
| }, | |
| // Retained observations are counted before temporal rejection. | |
| AliasSpec { | |
| canonical_group: 0, | |
| context_rule: None, | |
| valid_from_epoch: 10, | |
| valid_to_epoch: None, | |
| family: FAMILY_SP500, | |
| entity_kind: 0, | |
| unresolved_tag: false, | |
| }, | |
| ]; | |
| let exact_specs = ExactSpecs::new(Vec::new()).expect("empty exact specs are valid"); | |
| let mut scratch = ScanScratch::default(); | |
| let observed = scan_one( | |
| &automaton, | |
| &pattern_consumers, | |
| &consumers, | |
| &exact_specs, | |
| &aliases, | |
| &[], | |
| &[], | |
| &[], | |
| "alpha beta", | |
| 1, | |
| 0, | |
| &mut scratch, | |
| ) | |
| .expect("long aliases match"); | |
| assert_eq!(observed.1, ARTICLE_ACCEPTED); | |
| assert_eq!(observed.2, EXP_SUPPORTED); | |
| assert_eq!(observed.6, 2); | |
| } | |
| fn sorted_lexical_candidates_deduplicate_per_rule_and_span() { | |
| let automaton = build_automaton(&["signal".to_string()]).expect("valid automaton"); | |
| let pattern_consumers = vec![vec![0, 0]]; | |
| let consumers = vec![ConsumerSpec { | |
| kind: LEXICAL, | |
| a: 0, | |
| b: 0, | |
| c: 0, | |
| }]; | |
| let lexical = vec![LexicalSpec { | |
| context_window_chars: 0, | |
| context_terms: Vec::new(), | |
| is_event: true, | |
| market_wide_authorized: false, | |
| unresolved_tag: false, | |
| has_event_family: true, | |
| }]; | |
| let exact_specs = ExactSpecs::new(Vec::new()).expect("empty exact specs are valid"); | |
| let mut scratch = ScanScratch::default(); | |
| let observed = scan_one( | |
| &automaton, | |
| &pattern_consumers, | |
| &consumers, | |
| &exact_specs, | |
| &[], | |
| &lexical, | |
| &[], | |
| &[], | |
| "signal and signal", | |
| 1, | |
| 0, | |
| &mut scratch, | |
| ) | |
| .expect("lexical events match"); | |
| assert_eq!(observed.1, ARTICLE_ACCEPTED); | |
| assert_eq!(observed.6, 2); | |
| } | |
| fn conjunction_masks_count_once_and_do_not_leak_between_rows() { | |
| let automaton = build_automaton(&[ | |
| "commission".to_string(), | |
| "adopted".to_string(), | |
| "rule".to_string(), | |
| "market".to_string(), | |
| ]) | |
| .expect("valid automaton"); | |
| let pattern_consumers = vec![vec![0, 0], vec![1, 1], vec![2], vec![3]]; | |
| let consumers = vec![ | |
| ConsumerSpec { | |
| kind: CONJUNCTION, | |
| a: 0, | |
| b: 0, | |
| c: 0, | |
| }, | |
| ConsumerSpec { | |
| kind: CONJUNCTION, | |
| a: 0, | |
| b: 1, | |
| c: 0, | |
| }, | |
| ConsumerSpec { | |
| kind: CONJUNCTION, | |
| a: 0, | |
| b: 2, | |
| c: 0, | |
| }, | |
| ConsumerSpec { | |
| kind: CONJUNCTION, | |
| a: 0, | |
| b: 3, | |
| c: 0, | |
| }, | |
| ]; | |
| let exact_specs = ExactSpecs::new(vec![ExactSpec { | |
| value: "SEC".chars().collect(), | |
| mode: ExactMode::CaseSensitive, | |
| kind: CONJUNCTION, | |
| index: 0, | |
| }]) | |
| .expect("valid exact actor"); | |
| let conjunctions = vec![ConjunctionSpec { | |
| actor_phrase_count: 1, | |
| action_phrase_count: 1, | |
| object_phrase_count: 1, | |
| scope_phrase_count: 1, | |
| }]; | |
| let mut scratch = ScanScratch::default(); | |
| let scan = |text: &str, row_index: usize, scratch: &mut ScanScratch| { | |
| scan_one( | |
| &automaton, | |
| &pattern_consumers, | |
| &consumers, | |
| &exact_specs, | |
| &[], | |
| &[], | |
| &[], | |
| &conjunctions, | |
| text, | |
| 1, | |
| row_index, | |
| scratch, | |
| ) | |
| }; | |
| let first = scan( | |
| "SEC commission adopted adopted rule market", | |
| 0, | |
| &mut scratch, | |
| ) | |
| .expect("complete conjunction"); | |
| assert_eq!(first.6, 1); | |
| assert_eq!(first.5, EXP_SUPPORTED); | |
| assert!(scan("SEC adopted rule", 1, &mut scratch).is_none()); | |
| let third = scan("commission adopted rule market", 2, &mut scratch) | |
| .expect("later complete conjunction"); | |
| assert_eq!(third.6, 1); | |
| } | |
| // Block fixtures. Existing normalization_reference_checker is also | |
| // changed by the proposal to run each current scalar fixture in both modes. | |
| fn mixed_avx2_prefix_stops_at_original_unicode_boundaries() { | |
| println!("Phase31 block runtime AVX2 available: {}", mixed_ascii_avx2_available()); | |
| if !mixed_ascii_avx2_available() { | |
| return; // The portable/forced-scalar fixtures still run everywhere. | |
| } | |
| let pattern = b"A \t-`B/\0\x1c\x7f.,:;()[]{}"; | |
| for prefix_len in 0..=96 { | |
| for character in ['é', 'fi', 'İ', 'ı', 'ſ', 'K', '界', '🦀', '\u{fdfa}'] { | |
| let prefix: Vec<u8> = pattern.iter().copied().cycle().take(prefix_len).collect(); | |
| let mut text = String::from_utf8(prefix.clone()).unwrap(); | |
| text.push(character); | |
| text.push_str("ABCDEFGHIJKLMNOPQRSTUVWXYZ 0123456789"); | |
| for carry_space in [false, true] { | |
| // Nonzero base proves offsets are source positions, not | |
| // local block positions. A previous expansion is retained. | |
| let mut expected = b"fi".to_vec(); | |
| let mut expected_offsets = vec![5, 5]; | |
| let mut expected_pending = carry_space; | |
| for (index, byte) in prefix.iter().copied().enumerate() { | |
| normalize_source_char(byte as char, 9 + index, &mut expected, | |
| &mut expected_offsets, &mut expected_pending); | |
| } | |
| let mut observed = b"fi".to_vec(); | |
| let mut observed_offsets = vec![5, 5]; | |
| let mut observed_pending = carry_space; | |
| // SAFETY: runtime AVX2 check above; helper bounds every load. | |
| let consumed = unsafe { | |
| normalize_mixed_ascii_prefix_avx2_into( | |
| text.as_bytes(), 9, &mut observed, &mut observed_offsets, | |
| &mut observed_pending, | |
| ) | |
| }; | |
| assert_eq!(consumed, prefix_len, "prefix={prefix_len} char={character:?}"); | |
| assert_eq!(observed, expected); | |
| assert_eq!(observed_offsets, expected_offsets); | |
| assert_eq!(observed_pending, expected_pending); | |
| } | |
| } | |
| } | |
| // Every ASCII byte in every SIMD lane, including whitespace and | |
| // separators inside blocks. This does not rely on whitespace-free data. | |
| for lane in 0..96 { | |
| for byte in 0u8..=0x7f { | |
| let mut prefix = vec![b'x'; 96]; | |
| prefix[lane] = byte; | |
| let mut text = String::from_utf8(prefix.clone()).unwrap(); | |
| text.push_str("fiABCDEFGHIJKLMNOPQRSTUVWXYZ 0123456789"); | |
| let mut expected = Vec::new(); | |
| let mut expected_offsets = Vec::new(); | |
| let mut expected_pending = false; | |
| for (index, source_byte) in prefix.iter().copied().enumerate() { | |
| normalize_source_char(source_byte as char, 7 + index, &mut expected, | |
| &mut expected_offsets, &mut expected_pending); | |
| } | |
| let mut observed = Vec::new(); | |
| let mut observed_offsets = Vec::new(); | |
| let mut observed_pending = false; | |
| let consumed = unsafe { | |
| normalize_mixed_ascii_prefix_avx2_into( | |
| text.as_bytes(), 7, &mut observed, &mut observed_offsets, | |
| &mut observed_pending, | |
| ) | |
| }; | |
| assert_eq!(consumed, 96); | |
| assert_eq!(observed, expected, "byte={byte} lane={lane}"); | |
| assert_eq!(observed_offsets, expected_offsets, "byte={byte} lane={lane}"); | |
| assert_eq!(observed_pending, expected_pending, "byte={byte} lane={lane}"); | |
| } | |
| } | |
| // No padded/overreading load on short tails. Complete mixed fixtures | |
| // exercise the caller's scalar processing of these untouched bytes. | |
| for length in [0, 1, 15, 16, 31, 32, 33, 63, 64, 65] { | |
| let input = vec![b'A'; length]; | |
| let mut normalized = Vec::new(); | |
| let mut offsets = Vec::new(); | |
| let mut pending_space = false; | |
| let consumed = unsafe { | |
| normalize_mixed_ascii_prefix_avx2_into( | |
| &input, 7, &mut normalized, &mut offsets, &mut pending_space, | |
| ) | |
| }; | |
| assert_eq!(consumed, (length / 32) * 32); | |
| assert_eq!(normalized, vec![b'a'; consumed]); | |
| assert_eq!(offsets, (7..7 + consumed).collect::<Vec<_>>()); | |
| } | |
| } | |
| fn mixed_block_modes_collapse_whole_blocks_across_expansions() { | |
| let mut check = normalization_reference_checker(); | |
| for block_len in [32, 64] { | |
| // Every lane belongs to the collapse mask, including across two | |
| // consecutive blocks. Include spaces, controls and punctuation. | |
| let collapse: String = b" \t\n\r\x0b\x0c\x1c\x1d\x1e\x1f-.,:;()[]{}/" | |
| .iter().copied().cycle().take(block_len).map(char::from).collect(); | |
| for text in [ | |
| // No previous output: all-one masks must not create leading space. | |
| format!("{collapse}B\u{00a0}"), | |
| // Carry space through one/two masks after a Unicode expansion. | |
| format!("fi{collapse}B"), | |
| // Trailing masks must remain pending, never emitted at end-of-row. | |
| format!("fi{collapse}"), | |
| // A scalar Unicode space starts the same pending state before | |
| // the first vector block; repeated blocks must not duplicate it. | |
| format!("fi\u{00a0}{collapse}B"), | |
| ] { | |
| check(&text); | |
| } | |
| for allow_avx2 in [false, true] { | |
| let mut normalized = Vec::new(); | |
| let mut offsets = Vec::new(); | |
| let text = format!("fi{collapse}B"); | |
| let flags = normalize_text_into_with_mixed_avx2( | |
| &text, &mut normalized, &mut offsets, allow_avx2, | |
| ); | |
| assert_eq!(flags, (false, false, block_len + 2)); | |
| assert_eq!(normalized, b"fi b"); | |
| assert_eq!(offsets, [0, 0, block_len + 1, block_len + 1]); | |
| let text = format!("fi{collapse}"); | |
| normalize_text_into_with_mixed_avx2( | |
| &text, &mut normalized, &mut offsets, allow_avx2, | |
| ); | |
| assert_eq!(normalized, b"fi"); | |
| assert_eq!(offsets, [0, 0]); | |
| } | |
| } | |
| } | |
| fn mixed_block_modes_keep_context_windows_across_vector_edges() { | |
| let texts = [ | |
| format!("{}fi --\tİ{}\u{00a0}B", "A".repeat(31), "Z".repeat(33)), | |
| format!("\u{00a0}{}—{}\u{fdfa} ", "B`/ ".repeat(16), "C".repeat(31)), | |
| format!("{}Kıſ{}🦀{}", "A".repeat(32), "D".repeat(32), "E".repeat(33)), | |
| ]; | |
| for text in texts { | |
| let chars: Vec<char> = text.chars().collect(); | |
| for allow_avx2 in [false, true] { | |
| let mut normalized = Vec::new(); | |
| let mut offsets = Vec::new(); | |
| let flags = normalize_text_into_with_mixed_avx2( | |
| &text, &mut normalized, &mut offsets, allow_avx2, | |
| ); | |
| assert_eq!(flags.2, chars.len()); | |
| for start in [0, 1, 15, 16, 31, 32, 33, 63, 64, chars.len()] { | |
| let start = start.min(chars.len()); | |
| for span in [0, 1, 7] { | |
| let end = (start + span).min(chars.len()); | |
| for width in [0, 1, 2, 31, 32, 33, 65] { | |
| let left = start.saturating_sub(width); | |
| let right = end.saturating_add(width).min(chars.len()); | |
| let mut expected = Vec::new(); | |
| let mut expected_offsets = Vec::new(); | |
| normalize_chars_into(&chars[left..right], left, &mut expected, | |
| &mut expected_offsets); | |
| assert_eq!( | |
| normalized_context_window(&normalized, &offsets, chars.len(), | |
| start, end, width), | |
| expected.as_slice(), | |
| "allow_avx2={allow_avx2} start={start} end={end} width={width}", | |
| ); | |
| } | |
| } | |
| } | |
| } | |
| } | |
| } | |
| } | |