diff --git "a/rust/core.rs" "b/rust/core.rs" --- "a/rust/core.rs" +++ "b/rust/core.rs" @@ -1,1372 +1,2006 @@ -use aho_corasick::{AhoCorasick, AhoCorasickBuilder, MatchKind}; -use pyo3::exceptions::PyValueError; -use pyo3::prelude::*; -use rayon::prelude::*; -use rayon::ThreadPoolBuilder; -use std::collections::{HashMap, HashSet}; - -static UNICODE_INDEX: &[u8] = include_bytes!("generated/unicode_index.bin"); -static UNICODE_POOL: &[u8] = include_bytes!("generated/unicode_pool.bin"); -static PYTHON_WHITESPACE: &[u8] = include_bytes!("generated/python_whitespace.bin"); - -const ALIAS: u8 = 0; -const LEXICAL: u8 = 1; -const CONJUNCTION: u8 = 2; - -const FAMILY_NONE: i8 = -1; -const FAMILY_SP500: i8 = 0; -const FAMILY_NASDAQ100: i8 = 1; -const FAMILY_RUSSELL2000: i8 = 2; -const FAMILY_MARKET_WIDE: i8 = 3; - -const ARTICLE_ACCEPTED: u8 = 0; -const ARTICLE_REJECTED: u8 = 2; -const EXP_SUPPORTED: u8 = 0; -const EXP_PLAUSIBLE: u8 = 1; -const EXP_NOT_SUPPORTED: u8 = 2; - -#[derive(Clone)] -struct ConsumerSpec { - kind: u8, - a: usize, - b: usize, - c: usize, -} - -#[derive(Clone, Copy, Eq, PartialEq)] -enum ExactMode { - CaseSensitive, - PythonIgnoreCaseAsciiLiteral, -} - -#[derive(Clone)] -struct ExactSpec { - value: Vec, - mode: ExactMode, - kind: u8, - index: usize, -} - -struct ExactAutomaton { - automaton: AhoCorasick, - pattern_specs: Vec>, -} - -struct ExactSpecs { - specs: Vec, - case_sensitive: Option, - ignorecase_ascii: Option, -} - -#[derive(Clone, Copy)] -struct ExactByteMatch { - spec_index: usize, - start: usize, - end: usize, -} - -#[derive(Clone)] -struct ContextRule { - window_chars: usize, - accept_clauses: Vec>>, - reject_terms: Vec>, -} - -#[derive(Clone)] -struct AliasSpec { - canonical_group: usize, - context_rule: Option, - valid_from_epoch: i64, - valid_to_epoch: Option, - family: i8, - entity_kind: u8, - unresolved_tag: bool, -} - -#[derive(Clone)] -struct LexicalSpec { - context_window_chars: usize, - context_terms: Vec>, - is_event: bool, - market_wide_authorized: bool, - unresolved_tag: bool, - has_event_family: bool, -} - -#[derive(Clone)] -struct ConjunctionSpec { - actor_phrase_count: usize, - action_phrase_count: usize, - object_phrase_count: usize, - scope_phrase_count: usize, -} - -#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] -struct ObservationKey { - kind: u8, - index: usize, - start: usize, - end: usize, -} - -// (row_index, article_status, sp500, nasdaq100, russell2000, market_wide, -// observation_count, native_candidate, exact_candidate) -type FactRow = (usize, u8, u8, u8, u8, u8, usize, bool, bool); - -#[derive(Default)] -struct ScanScratch { - chars: Vec, - normalized: Vec, - offsets: Vec, - window: Vec, - window_offsets: Vec, - exact_byte_matches: Vec, - exact_matches: Vec<(usize, usize, usize)>, - exact_boundaries: Vec, - alias_candidates: Vec<(usize, usize, usize)>, - lexical_candidates: Vec<(usize, usize, usize)>, - conjunction_hits: HashSet<(usize, usize, usize)>, - longest: HashMap<(usize, usize), usize>, - observations: HashSet, -} - -impl ScanScratch { - fn reset(&mut self) { - self.chars.clear(); - self.normalized.clear(); - self.offsets.clear(); - self.window.clear(); - self.window_offsets.clear(); - self.exact_byte_matches.clear(); - self.exact_matches.clear(); - self.exact_boundaries.clear(); - self.alias_candidates.clear(); - self.lexical_candidates.clear(); - self.conjunction_hits.clear(); - self.longest.clear(); - self.observations.clear(); - } -} - -#[inline(always)] -fn is_python_ascii_space(byte: u8) -> bool { - matches!( - byte, - b' ' | b'\t' | b'\n' | 0x0b | 0x0c | b'\r' | 0x1c..=0x1f - ) -} - -#[inline(always)] -fn is_separator_ascii(byte: u8) -> bool { - matches!( - byte, - b'-' | b'.' | b',' | b':' | b';' | b'(' | b')' | b'[' | b']' | b'{' | b'}' | b'/' - ) -} - -#[inline(always)] -fn is_token_byte(byte: u8) -> bool { - byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'_' -} - -#[inline(always)] -fn is_exact_token_char(ch: char, mode: ExactMode) -> bool { - if ch.is_ascii_alphanumeric() || ch == '_' { - return true; - } - // Python's Unicode-aware re.IGNORECASE expands [A-Za-z] to these four - // additional code points. They are token characters only for the - // python_ignorecase_ascii_literal mode; applying this globally would change - // the frozen boundary behavior of case-sensitive exact aliases. - matches!(mode, ExactMode::PythonIgnoreCaseAsciiLiteral) - && matches!(ch, '\u{0130}' | '\u{0131}' | '\u{017f}' | '\u{212a}') -} - -#[inline(always)] -fn is_python_space(ch: char) -> bool { - let cp = ch as usize; - let byte = PYTHON_WHITESPACE[cp >> 3]; - (byte & (1 << (cp & 7))) != 0 -} - -#[inline(always)] -fn mapped_bytes(ch: char) -> Option<&'static [u8]> { - let cp = ch as usize; - let base = cp * 4; - let offset = u32::from_le_bytes([ - UNICODE_INDEX[base], - UNICODE_INDEX[base + 1], - UNICODE_INDEX[base + 2], - UNICODE_INDEX[base + 3], - ]) as usize; - if offset == 0 { - return None; - } - let len = u16::from_le_bytes([UNICODE_POOL[offset], UNICODE_POOL[offset + 1]]) as usize; - Some(&UNICODE_POOL[offset + 2..offset + 2 + len]) -} - -#[inline(always)] -fn normalize_post_char( - ch: char, - source_index: usize, - normalized: &mut Vec, - offsets: &mut Vec, - pending_space: &mut bool, -) { - let ch = match ch { - '-' | '‐' | '‑' | '‒' | '–' | '—' | '―' | '−' => ' ', - '.' | ',' | ':' | ';' | '(' | ')' | '[' | ']' | '{' | '}' | '/' | '/' => ' ', - '\'' | '’' | '‘' | '‛' | '`' | '´' => '\'', - other => other, - }; - if is_python_space(ch) { - *pending_space = !normalized.is_empty(); - return; - } - if *pending_space { - normalized.push(b' '); - offsets.push(source_index); - *pending_space = false; - } - let mut buffer = [0u8; 4]; - let encoded = ch.encode_utf8(&mut buffer).as_bytes(); - normalized.extend_from_slice(encoded); - offsets.extend(std::iter::repeat(source_index).take(encoded.len())); -} - -fn normalize_source_char( - source_character: char, - source_index: usize, - normalized: &mut Vec, - offsets: &mut Vec, - pending_space: &mut bool, -) { - if source_character.is_ascii() { - let mut byte = (source_character as u8).to_ascii_lowercase(); - if is_separator_ascii(byte) { - byte = b' '; - } else if byte == b'`' { - byte = b'\''; - } - if is_python_ascii_space(byte) { - *pending_space = !normalized.is_empty(); - return; - } - if *pending_space { - normalized.push(b' '); - offsets.push(source_index); - *pending_space = false; - } - normalized.push(byte); - offsets.push(source_index); - } else if let Some(mapped) = mapped_bytes(source_character) { - let mapped = std::str::from_utf8(mapped).expect("generated Unicode pool is valid UTF-8"); - for character in mapped.chars() { - normalize_post_char(character, source_index, normalized, offsets, pending_space); - } - } else { - normalize_post_char( - source_character, - source_index, - normalized, - offsets, - pending_space, - ); - } -} - -fn normalize_chars_into( - chars: &[char], - source_base: usize, - normalized: &mut Vec, - offsets: &mut Vec, -) { - normalized.clear(); - offsets.clear(); - normalized.reserve(chars.len()); - offsets.reserve(chars.len()); - let mut pending_space = false; - for (local_index, source_character) in chars.iter().copied().enumerate() { - let source_index = source_base + local_index; - normalize_source_char( - source_character, - source_index, - normalized, - offsets, - &mut pending_space, - ); - } -} - -#[inline(always)] -fn normalize_ascii_byte(mut byte: u8) -> u8 { - byte.make_ascii_lowercase(); - if is_separator_ascii(byte) { - b' ' - } else if byte == b'`' { - b'\'' - } else { - byte - } -} - -fn normalize_ascii_text_into(text: &str, normalized: &mut Vec, offsets: &mut Vec) { - debug_assert!(text.is_ascii()); - let mut pending_space = false; - for (source_index, source_byte) in text.bytes().enumerate() { - let byte = normalize_ascii_byte(source_byte); - if is_python_ascii_space(byte) { - pending_space = !normalized.is_empty(); - continue; - } - if pending_space { - normalized.push(b' '); - offsets.push(source_index); - pending_space = false; - } - normalized.push(byte); - offsets.push(source_index); - } -} - -fn normalize_text_into( - text: &str, - normalized: &mut Vec, - offsets: &mut Vec, -) -> (bool, bool) { - normalized.clear(); - offsets.clear(); - normalized.reserve(text.len()); - offsets.reserve(text.len()); - if text.is_ascii() { - normalize_ascii_text_into(text, normalized, offsets); - return (true, false); - } - let mut pending_space = false; - let mut all_ascii = true; - let mut exceptional_ignorecase = false; - for (source_index, source_character) in text.chars().enumerate() { - all_ascii &= source_character.is_ascii(); - exceptional_ignorecase |= matches!( - source_character, - '\u{0130}' | '\u{0131}' | '\u{017f}' | '\u{212a}' - ); - normalize_source_char( - source_character, - source_index, - normalized, - offsets, - &mut pending_space, - ); - } - (all_ascii, exceptional_ignorecase) -} - -#[inline] -fn normalized_contains(haystack: &[u8], needle: &[u8]) -> bool { - if needle.is_empty() || haystack.len() < needle.len() { - return false; - } - let mut cursor = 0usize; - while cursor + needle.len() <= haystack.len() { - if let Some(relative) = haystack[cursor..] - .windows(needle.len()) - .position(|window| window == needle) - { - let start = cursor + relative; - let end = start + needle.len(); - let left_ok = start == 0 || !is_token_byte(haystack[start - 1]); - let right_ok = end == haystack.len() || !is_token_byte(haystack[end]); - if left_ok && right_ok { - return true; - } - cursor = start + 1; - } else { - return false; - } - } - false -} - -fn normalized_window_into( - chars: &[char], - start: usize, - end: usize, - width: usize, - normalized: &mut Vec, - offsets: &mut Vec, -) { - let left = start.saturating_sub(width); - let right = (end + width).min(chars.len()); - normalize_chars_into(&chars[left..right], left, normalized, offsets); -} - -fn context_status( - chars: &[char], - start: usize, - end: usize, - rule: &ContextRule, - window: &mut Vec, - window_offsets: &mut Vec, -) -> u8 { - // 0=satisfied, 1=explicit_collision_exclusion, 2=context_clause_not_satisfied - normalized_window_into(chars, start, end, rule.window_chars, window, window_offsets); - if rule - .reject_terms - .iter() - .any(|term| normalized_contains(window, term)) - { - return 1; - } - if rule - .accept_clauses - .iter() - .any(|clause| clause.iter().all(|term| normalized_contains(window, term))) - { - 0 - } else { - 2 - } -} - -#[inline(always)] -fn py_ignorecase_ascii_literal_eq(pattern: char, observed: char) -> bool { - let lower = pattern.to_ascii_lowercase(); - if observed.is_ascii() { - return observed.to_ascii_lowercase() == lower; - } - match lower { - 'i' => matches!(observed, 'İ' | 'ı'), - 's' => observed == 'ſ', - 'k' => observed == 'K', - _ => false, - } -} - -fn exact_matches(chars: &[char], spec: &ExactSpec) -> Vec<(usize, usize)> { - let width = spec.value.len(); - if width == 0 || width > chars.len() { - return Vec::new(); - } - let mut output = Vec::new(); - for start in 0..=chars.len() - width { - let end = start + width; - if start > 0 && is_exact_token_char(chars[start - 1], spec.mode) { - continue; - } - if end < chars.len() && is_exact_token_char(chars[end], spec.mode) { - continue; - } - let matched = spec.value.iter().enumerate().all(|(offset, expected)| { - let observed = chars[start + offset]; - match spec.mode { - ExactMode::CaseSensitive => observed == *expected, - ExactMode::PythonIgnoreCaseAsciiLiteral => { - py_ignorecase_ascii_literal_eq(*expected, observed) - } - } - }); - if matched { - output.push((start, end)); - } - } - output -} - -#[inline(always)] -fn exact_boundaries_match(text: &str, start: usize, end: usize, mode: ExactMode) -> bool { - let left_ok = start == 0 - || !is_exact_token_char( - text[..start] - .chars() - .next_back() - .expect("nonempty exact left boundary"), - mode, - ); - let right_ok = end == text.len() - || !is_exact_token_char( - text[end..] - .chars() - .next() - .expect("nonempty exact right boundary"), - mode, - ); - left_ok && right_ok -} - -impl ExactAutomaton { - fn append_matches(&self, text: &str, mode: ExactMode, output: &mut Vec) { - for matched in self.automaton.find_overlapping_iter(text.as_bytes()) { - let start = matched.start(); - let end = matched.end(); - if !exact_boundaries_match(text, start, end, mode) { - continue; - } - for &spec_index in &self.pattern_specs[matched.pattern().as_usize()] { - output.push(ExactByteMatch { - spec_index, - start, - end, - }); - } - } - } -} - -fn build_exact_automaton( - specs: &[ExactSpec], - mode: ExactMode, - ascii_case_insensitive: bool, -) -> PyResult> { - let mut patterns = Vec::::new(); - let mut pattern_specs = Vec::>::new(); - let mut pattern_ids = HashMap::::new(); - - for (spec_index, spec) in specs.iter().enumerate() { - if spec.mode != mode { - continue; - } - let value: String = spec.value.iter().collect(); - let pattern_key = if ascii_case_insensitive { - value.to_ascii_lowercase() - } else { - value.clone() - }; - if let Some(&pattern_id) = pattern_ids.get(&pattern_key) { - pattern_specs[pattern_id].push(spec_index); - } else { - let pattern_id = patterns.len(); - pattern_ids.insert(pattern_key, pattern_id); - patterns.push(value); - pattern_specs.push(vec![spec_index]); - } - } - - if patterns.is_empty() { - return Ok(None); - } - let automaton = AhoCorasickBuilder::new() - .match_kind(MatchKind::Standard) - .ascii_case_insensitive(ascii_case_insensitive) - .build(&patterns) - .map_err(|error| PyValueError::new_err(error.to_string()))?; - Ok(Some(ExactAutomaton { - automaton, - pattern_specs, - })) -} - -impl ExactSpecs { - fn new(specs: Vec) -> PyResult { - let case_sensitive = build_exact_automaton(&specs, ExactMode::CaseSensitive, false)?; - let ignorecase_ascii = - build_exact_automaton(&specs, ExactMode::PythonIgnoreCaseAsciiLiteral, true)?; - Ok(Self { - specs, - case_sensitive, - ignorecase_ascii, - }) - } - - fn append_matching( - &self, - text: &str, - byte_matches: &mut Vec, - output: &mut Vec<(usize, usize, usize)>, - exceptional_chars: &mut Vec, - boundaries: &mut Vec, - text_is_ascii: bool, - exceptional_ignorecase: bool, - ) { - byte_matches.clear(); - output.clear(); - exceptional_chars.clear(); - boundaries.clear(); - if let Some(automaton) = &self.case_sensitive { - automaton.append_matches(text, ExactMode::CaseSensitive, byte_matches); - } - - if !exceptional_ignorecase { - if let Some(automaton) = &self.ignorecase_ascii { - automaton.append_matches( - text, - ExactMode::PythonIgnoreCaseAsciiLiteral, - byte_matches, - ); - } - } - - output.reserve(byte_matches.len()); - if text_is_ascii { - output.extend( - byte_matches - .iter() - .copied() - .map(|matched| (matched.spec_index, matched.start, matched.end)), - ); - } else if !byte_matches.is_empty() { - // Aho-Corasick reports UTF-8 byte offsets while the frozen Python - // surface reports code-point offsets. Build this map only for the - // uncommon non-ASCII row that actually produced an exact hit. - boundaries.extend(text.char_indices().map(|(offset, _)| offset)); - boundaries.push(text.len()); - output.extend(byte_matches.iter().copied().map(|matched| { - let start = boundaries - .binary_search(&matched.start) - .expect("exact match starts at a UTF-8 character boundary"); - let end = boundaries - .binary_search(&matched.end) - .expect("exact match ends at a UTF-8 character boundary"); - (matched.spec_index, start, end) - })); - } - - if exceptional_ignorecase { - // Python re.IGNORECASE gives ASCII I/S/K four additional Unicode - // equivalents (İ, ı, ſ, K). The common ASCII automaton cannot change - // byte width while matching, so retain the exact scalar fallback - // only for rows that contain one of those exceptional code points. - exceptional_chars.extend(text.chars()); - for (spec_index, spec) in self.specs.iter().enumerate() { - if spec.mode != ExactMode::PythonIgnoreCaseAsciiLiteral { - continue; - } - output.extend( - exact_matches(exceptional_chars, spec) - .into_iter() - .map(|(start, end)| (spec_index, start, end)), - ); - } - } - } - - #[cfg(test)] - fn matching(&self, text: &str) -> Vec<(usize, usize, usize)> { - let mut byte_matches = Vec::new(); - let mut output = Vec::new(); - let mut exceptional_chars = Vec::new(); - let mut boundaries = Vec::new(); - let mut normalized = Vec::new(); - let mut offsets = Vec::new(); - let (text_is_ascii, exceptional_ignorecase) = - normalize_text_into(text, &mut normalized, &mut offsets); - self.append_matching( - text, - &mut byte_matches, - &mut output, - &mut exceptional_chars, - &mut boundaries, - text_is_ascii, - exceptional_ignorecase, - ); - output - } -} - -fn build_automaton(patterns: &[String]) -> PyResult { - if patterns.iter().any(|pattern| pattern.is_empty()) { - return Err(PyValueError::new_err("patterns must be non-empty")); - } - AhoCorasickBuilder::new() - .match_kind(MatchKind::Standard) - .build(patterns) - .map_err(|error| PyValueError::new_err(error.to_string())) -} - -fn scan_one( - automaton: &AhoCorasick, - pattern_consumers: &[Vec], - consumers: &[ConsumerSpec], - exact_specs: &ExactSpecs, - aliases: &[AliasSpec], - lexical: &[LexicalSpec], - contexts: &[ContextRule], - conjunctions: &[ConjunctionSpec], - text: &str, - observed_epoch: i64, - row_index: usize, - scratch: &mut ScanScratch, -) -> Option { - scratch.reset(); - let (text_is_ascii, exceptional_ignorecase) = - normalize_text_into(text, &mut scratch.normalized, &mut scratch.offsets); - let mut native_candidate = false; - if !scratch.normalized.is_empty() { - for matched in automaton.find_overlapping_iter(&scratch.normalized) { - let start = matched.start(); - let end = matched.end(); - let left_ok = start == 0 || !is_token_byte(scratch.normalized[start - 1]); - let right_ok = - end == scratch.normalized.len() || !is_token_byte(scratch.normalized[end]); - if !left_ok || !right_ok { - continue; - } - native_candidate = true; - let original_start = scratch.offsets[start]; - let original_end = scratch.offsets[end - 1] + 1; - for &consumer_id in &pattern_consumers[matched.pattern().as_usize()] { - let spec = &consumers[consumer_id]; - match spec.kind { - ALIAS => { - scratch - .alias_candidates - .push((spec.a, original_start, original_end)); - } - LEXICAL => { - scratch - .lexical_candidates - .push((spec.a, original_start, original_end)); - } - CONJUNCTION => { - scratch.conjunction_hits.insert((spec.a, spec.b, spec.c)); - } - _ => unreachable!("validated consumer kind"), - } - } - } - } - - let mut exact_candidate = false; - exact_specs.append_matching( - text, - &mut scratch.exact_byte_matches, - &mut scratch.exact_matches, - &mut scratch.chars, - &mut scratch.exact_boundaries, - text_is_ascii, - exceptional_ignorecase, - ); - for &(spec_index, start, end) in &scratch.exact_matches { - exact_candidate = true; - let spec = &exact_specs.specs[spec_index]; - match spec.kind { - ALIAS => scratch.alias_candidates.push((spec.index, start, end)), - LEXICAL => scratch.lexical_candidates.push((spec.index, start, end)), - CONJUNCTION => { - scratch.conjunction_hits.insert((spec.index, 0, usize::MAX)); - } - _ => unreachable!("validated exact kind"), - } - } - - if scratch.alias_candidates.is_empty() - && scratch.lexical_candidates.is_empty() - && scratch.conjunction_hits.is_empty() - { - return None; - } - - // Normalized scanning and the exact automata do not need a materialized - // Unicode scalar vector. Build it only after the row produced a consumer; - // the common no-fact path therefore performs one traversal and no char Vec - // allocation. Exceptional Python ignorecase rows may already have filled it. - if scratch.chars.is_empty() && !text.is_empty() { - scratch.chars.extend(text.chars()); - } - - // Longest alias per (canonical_id, start), preserving every alias that ties. - for &(alias_index, start, end) in &scratch.alias_candidates { - let key = (aliases[alias_index].canonical_group, start); - scratch - .longest - .entry(key) - .and_modify(|value| *value = (*value).max(end)) - .or_insert(end); - } - - let mut accepted_nonsemantic = false; - let mut accepted_evidence = false; - let mut unresolved_exposure = false; - let mut unresolved_event = false; - let mut supported_mask: u8 = 0; - - for candidate_index in 0..scratch.alias_candidates.len() { - let (alias_index, start, end) = scratch.alias_candidates[candidate_index]; - let spec = &aliases[alias_index]; - if end < scratch.longest[&(spec.canonical_group, start)] { - continue; - } - let key = ObservationKey { - kind: ALIAS, - index: alias_index, - start, - end, - }; - if !scratch.observations.insert(key) { - continue; - } - - let active = observed_epoch >= spec.valid_from_epoch - && spec - .valid_to_epoch - .map(|until| observed_epoch < until) - .unwrap_or(true); - if !active { - continue; - } - - let context_code = if let Some(rule_index) = spec.context_rule { - context_status( - &scratch.chars, - start, - end, - &contexts[rule_index], - &mut scratch.window, - &mut scratch.window_offsets, - ) - } else { - 0 - }; - if context_code != 0 { - // Frozen exposure unresolved filter excludes explicit collision and - // context-clause failures. Missing context cannot occur because the - // validated config rejects unknown rules during construction. - continue; - } - - accepted_nonsemantic = true; - if spec.family != FAMILY_NONE && (spec.entity_kind == 0 || spec.entity_kind == 1) { - accepted_evidence = true; - supported_mask |= 1u8 << (spec.family as u8); - } else if spec.unresolved_tag { - unresolved_exposure = true; - } - } - - for candidate_index in 0..scratch.lexical_candidates.len() { - let (lexical_index, start, end) = scratch.lexical_candidates[candidate_index]; - let spec = &lexical[lexical_index]; - let key = ObservationKey { - kind: LEXICAL, - index: lexical_index, - start, - end, - }; - if !scratch.observations.insert(key) { - continue; - } - - let context_ok = if spec.context_terms.is_empty() { - true - } else { - normalized_window_into( - &scratch.chars, - start, - end, - spec.context_window_chars, - &mut scratch.window, - &mut scratch.window_offsets, - ); - spec.context_terms - .iter() - .any(|term| normalized_contains(&scratch.window, term)) - }; - - if spec.is_event { - if spec.market_wide_authorized { - accepted_evidence = true; - supported_mask |= 1u8 << FAMILY_MARKET_WIDE; - } else { - if spec.unresolved_tag { - unresolved_exposure = true; - } - if spec.has_event_family { - unresolved_event = true; - } - } - } else { - if context_ok { - accepted_nonsemantic = true; - } - if spec.market_wide_authorized && context_ok { - accepted_evidence = true; - supported_mask |= 1u8 << FAMILY_MARKET_WIDE; - } else if spec.unresolved_tag { - // Matches frozen generic lexical unresolved branch, including a - // lexical context failure, whose evidence reason is the generic - // identity-resolved-without-path code. - unresolved_exposure = true; - } - } - } - - for (rule_index, rule) in conjunctions.iter().enumerate() { - let field_has = |field: usize, count: usize| -> bool { - (0..count).any(|phrase_index| { - scratch - .conjunction_hits - .contains(&(rule_index, field, phrase_index)) - }) - }; - let actor_phrase = field_has(0, rule.actor_phrase_count); - let actor_exact = scratch - .conjunction_hits - .contains(&(rule_index, 0, usize::MAX)); - let actor = actor_phrase || actor_exact; - let action = field_has(1, rule.action_phrase_count); - let object = field_has(2, rule.object_phrase_count); - let scope = field_has(3, rule.scope_phrase_count); - if actor && action && object && scope { - // Only one conjunction observation exists per configured rule. - scratch.observations.insert(ObservationKey { - kind: CONJUNCTION, - index: rule_index, - start: 0, - end: 1, - }); - accepted_evidence = true; - supported_mask |= 1u8 << FAMILY_MARKET_WIDE; - } - } - - if scratch.observations.is_empty() { - return None; - } - let article_status = if accepted_evidence || accepted_nonsemantic || unresolved_event { - ARTICLE_ACCEPTED - } else { - ARTICLE_REJECTED - }; - let completed_elsewhere = supported_mask != 0; - let exposure_status = |family: i8| -> u8 { - if (supported_mask & (1u8 << (family as u8))) != 0 { - EXP_SUPPORTED - } else if family != FAMILY_MARKET_WIDE && !completed_elsewhere && unresolved_exposure { - EXP_PLAUSIBLE - } else { - EXP_NOT_SUPPORTED - } - }; - - Some(( - row_index, - article_status, - exposure_status(FAMILY_SP500), - exposure_status(FAMILY_NASDAQ100), - exposure_status(FAMILY_RUSSELL2000), - exposure_status(FAMILY_MARKET_WIDE), - scratch.observations.len(), - native_candidate, - exact_candidate, - )) -} - +use aho_corasick::{AhoCorasick, AhoCorasickBuilder, AhoCorasickKind, MatchKind}; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::*; +use rayon::prelude::*; +use rayon::ThreadPoolBuilder; +use std::collections::HashMap; + +static UNICODE_INDEX: &[u8] = include_bytes!("generated/unicode_index.bin"); +static UNICODE_POOL: &[u8] = include_bytes!("generated/unicode_pool.bin"); +static PYTHON_WHITESPACE: &[u8] = include_bytes!("generated/python_whitespace.bin"); + +const ALIAS: u8 = 0; +const LEXICAL: u8 = 1; +const CONJUNCTION: u8 = 2; + +const FAMILY_NONE: i8 = -1; +const FAMILY_SP500: i8 = 0; +const FAMILY_NASDAQ100: i8 = 1; +const FAMILY_RUSSELL2000: i8 = 2; +const FAMILY_MARKET_WIDE: i8 = 3; + +const ARTICLE_ACCEPTED: u8 = 0; +const ARTICLE_REJECTED: u8 = 2; +const EXP_SUPPORTED: u8 = 0; +const EXP_PLAUSIBLE: u8 = 1; +const EXP_NOT_SUPPORTED: u8 = 2; + +const NORMALIZED_DFA_MAX_PATTERNS: usize = 512; +const NORMALIZED_DFA_MAX_PATTERN_BYTES: usize = 16 * 1024; +const NORMALIZED_DFA_MAX_HEAP_BYTES: usize = 32 * 1024 * 1024; + +fn try_standard_automaton( + patterns: &[String], + ascii_case_insensitive: bool, + kind: Option, +) -> Result { + if patterns.iter().any(|pattern| pattern.is_empty()) { + return Err("patterns must be non-empty".to_string()); + } + let mut builder = AhoCorasickBuilder::new(); + builder + .match_kind(MatchKind::Standard) + .ascii_case_insensitive(ascii_case_insensitive) + .prefilter(true); + if let Some(kind) = kind { + builder.kind(Some(kind)); + } + builder.build(patterns).map_err(|error| error.to_string()) +} + +fn build_auto_automaton( + patterns: &[String], + ascii_case_insensitive: bool, +) -> PyResult { + try_standard_automaton(patterns, ascii_case_insensitive, None) + .map_err(PyValueError::new_err) +} + +fn normalized_dfa_eligible(patterns: &[String]) -> bool { + patterns.len() <= NORMALIZED_DFA_MAX_PATTERNS + && patterns + .iter() + .try_fold(0usize, |total, pattern| total.checked_add(pattern.len())) + .is_some_and(|total| total <= NORMALIZED_DFA_MAX_PATTERN_BYTES) +} + +fn build_automaton(patterns: &[String]) -> PyResult { + if normalized_dfa_eligible(patterns) { + if let Ok(dfa) = + try_standard_automaton(patterns, false, Some(AhoCorasickKind::DFA)) + { + if dfa.memory_usage() <= NORMALIZED_DFA_MAX_HEAP_BYTES { + return Ok(dfa); + } + } + } + build_auto_automaton(patterns, false) +} + +#[derive(Clone)] +struct ConsumerSpec { + kind: u8, + a: usize, + b: usize, + c: usize, +} + +#[derive(Clone, Copy, Eq, PartialEq)] +enum ExactMode { + CaseSensitive, + PythonIgnoreCaseAsciiLiteral, +} + +#[derive(Clone)] +struct ExactSpec { + value: Vec, + mode: ExactMode, + kind: u8, + index: usize, +} + +struct ExactAutomaton { + automaton: AhoCorasick, + pattern_specs: Vec>, +} + +struct ExactSpecs { + specs: Vec, + case_sensitive: Option, + ignorecase_ascii: Option, +} + +#[derive(Clone, Copy)] +struct ExactByteMatch { + spec_index: usize, + start: usize, + end: usize, +} + +#[derive(Clone)] +struct ContextRule { + window_chars: usize, + accept_clauses: Vec>>, + reject_terms: Vec>, +} + +#[derive(Clone)] +struct AliasSpec { + canonical_group: usize, + context_rule: Option, + valid_from_epoch: i64, + valid_to_epoch: Option, + family: i8, + entity_kind: u8, + unresolved_tag: bool, +} + +#[derive(Clone)] +struct LexicalSpec { + context_window_chars: usize, + context_terms: Vec>, + is_event: bool, + market_wide_authorized: bool, + unresolved_tag: bool, + has_event_family: bool, +} + +#[derive(Clone)] +struct ConjunctionSpec { + actor_phrase_count: usize, + action_phrase_count: usize, + object_phrase_count: usize, + scope_phrase_count: usize, +} + +impl ConjunctionSpec { + #[inline(always)] + fn contains_phrase(&self, field: usize, phrase_index: usize) -> bool { + let count = match field { + 0 => self.actor_phrase_count, + 1 => self.action_phrase_count, + 2 => self.object_phrase_count, + 3 => self.scope_phrase_count, + _ => return false, + }; + phrase_index < count + } +} + +#[derive(Default)] +struct ConjunctionMarks { + generation: u32, + stamps: Vec, + masks: Vec, + touched: Vec, +} + +impl ConjunctionMarks { + fn begin_row(&mut self, rule_count: usize) { + if self.stamps.len() < rule_count { + self.stamps.resize(rule_count, 0); + self.masks.resize(rule_count, 0); + } + self.touched.clear(); + self.generation = self.generation.wrapping_add(1); + if self.generation == 0 { + self.stamps.fill(0); + self.generation = 1; + } + } + + #[inline(always)] + fn mark(&mut self, rule_index: usize, field: usize) { + if rule_index >= self.stamps.len() || field >= 4 { + return; + } + if self.stamps[rule_index] != self.generation { + self.stamps[rule_index] = self.generation; + self.masks[rule_index] = 0; + self.touched.push(rule_index); + } + self.masks[rule_index] |= 1u8 << field; + } + + #[inline(always)] + fn is_complete(&self, rule_index: usize) -> bool { + self.stamps[rule_index] == self.generation && self.masks[rule_index] == 0b1111 + } + + #[inline(always)] + fn is_empty(&self) -> bool { + self.touched.is_empty() + } +} + +// (row_index, article_status, sp500, nasdaq100, russell2000, market_wide, +// observation_count, native_candidate, exact_candidate) +type FactRow = (usize, u8, u8, u8, u8, u8, usize, bool, bool); + +#[derive(Default)] +struct ScanScratch { + chars: Vec, + normalized: Vec, + offsets: Vec, + exact_byte_matches: Vec, + exact_matches: Vec<(usize, usize, usize)>, + exact_boundaries: Vec, + alias_candidates: Vec<(usize, usize, usize)>, + lexical_candidates: Vec<(usize, usize, usize)>, + conjunction_marks: ConjunctionMarks, +} + +impl ScanScratch { + fn reset(&mut self, conjunction_count: usize) { + self.chars.clear(); + self.normalized.clear(); + self.offsets.clear(); + self.exact_byte_matches.clear(); + self.exact_matches.clear(); + self.exact_boundaries.clear(); + self.alias_candidates.clear(); + self.lexical_candidates.clear(); + self.conjunction_marks.begin_row(conjunction_count); + } +} + +#[inline(always)] +fn is_python_ascii_space(byte: u8) -> bool { + matches!( + byte, + b' ' | b'\t' | b'\n' | 0x0b | 0x0c | b'\r' | 0x1c..=0x1f + ) +} + +#[inline(always)] +fn is_separator_ascii(byte: u8) -> bool { + matches!( + byte, + b'-' | b'.' | b',' | b':' | b';' | b'(' | b')' | b'[' | b']' | b'{' | b'}' | b'/' + ) +} + +#[inline(always)] +fn is_token_byte(byte: u8) -> bool { + byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'_' +} + +#[inline(always)] +fn is_exact_token_char(ch: char, mode: ExactMode) -> bool { + if ch.is_ascii_alphanumeric() || ch == '_' { + return true; + } + // Python's Unicode-aware re.IGNORECASE expands [A-Za-z] to these four + // additional code points. They are token characters only for the + // python_ignorecase_ascii_literal mode; applying this globally would change + // the frozen boundary behavior of case-sensitive exact aliases. + matches!(mode, ExactMode::PythonIgnoreCaseAsciiLiteral) + && matches!(ch, '\u{0130}' | '\u{0131}' | '\u{017f}' | '\u{212a}') +} + +#[inline(always)] +fn is_python_space(ch: char) -> bool { + let cp = ch as usize; + let byte = PYTHON_WHITESPACE[cp >> 3]; + (byte & (1 << (cp & 7))) != 0 +} + +#[inline(always)] +fn mapped_bytes(ch: char) -> Option<&'static [u8]> { + let cp = ch as usize; + let base = cp * 4; + let offset = u32::from_le_bytes([ + UNICODE_INDEX[base], + UNICODE_INDEX[base + 1], + UNICODE_INDEX[base + 2], + UNICODE_INDEX[base + 3], + ]) as usize; + if offset == 0 { + return None; + } + let len = u16::from_le_bytes([UNICODE_POOL[offset], UNICODE_POOL[offset + 1]]) as usize; + Some(&UNICODE_POOL[offset + 2..offset + 2 + len]) +} + +#[inline(always)] +fn normalize_post_char( + ch: char, + source_index: usize, + normalized: &mut Vec, + offsets: &mut Vec, + pending_space: &mut bool, +) { + let ch = match ch { + '-' | '‐' | '‑' | '‒' | '–' | '—' | '―' | '−' => ' ', + '.' | ',' | ':' | ';' | '(' | ')' | '[' | ']' | '{' | '}' | '/' | '/' => ' ', + '\'' | '’' | '‘' | '‛' | '`' | '´' => '\'', + other => other, + }; + if is_python_space(ch) { + *pending_space = !normalized.is_empty(); + return; + } + if *pending_space { + normalized.push(b' '); + offsets.push(source_index); + *pending_space = false; + } + let mut buffer = [0u8; 4]; + let encoded = ch.encode_utf8(&mut buffer).as_bytes(); + normalized.extend_from_slice(encoded); + offsets.extend(std::iter::repeat(source_index).take(encoded.len())); +} + +fn normalize_source_char( + source_character: char, + source_index: usize, + normalized: &mut Vec, + offsets: &mut Vec, + pending_space: &mut bool, +) { + if source_character.is_ascii() { + let mut byte = (source_character as u8).to_ascii_lowercase(); + if is_separator_ascii(byte) { + byte = b' '; + } else if byte == b'`' { + byte = b'\''; + } + if is_python_ascii_space(byte) { + *pending_space = !normalized.is_empty(); + return; + } + if *pending_space { + normalized.push(b' '); + offsets.push(source_index); + *pending_space = false; + } + normalized.push(byte); + offsets.push(source_index); + } else if let Some(mapped) = mapped_bytes(source_character) { + let mapped = std::str::from_utf8(mapped).expect("generated Unicode pool is valid UTF-8"); + for character in mapped.chars() { + normalize_post_char(character, source_index, normalized, offsets, pending_space); + } + } else { + normalize_post_char( + source_character, + source_index, + normalized, + offsets, + pending_space, + ); + } +} + +#[cfg(test)] +fn normalize_chars_into( + chars: &[char], + source_base: usize, + normalized: &mut Vec, + offsets: &mut Vec, +) { + normalized.clear(); + offsets.clear(); + normalized.reserve(chars.len()); + offsets.reserve(chars.len()); + let mut pending_space = false; + for (local_index, source_character) in chars.iter().copied().enumerate() { + let source_index = source_base + local_index; + normalize_source_char( + source_character, + source_index, + normalized, + offsets, + &mut pending_space, + ); + } +} + +#[inline(always)] +fn normalize_ascii_byte(mut byte: u8) -> u8 { + byte.make_ascii_lowercase(); + if is_separator_ascii(byte) { + b' ' + } else if byte == b'`' { + b'\'' + } else { + byte + } +} + +fn normalize_ascii_text_into(text: &str, normalized: &mut Vec, offsets: &mut Vec) { + debug_assert!(text.is_ascii()); + let mut pending_space = false; + for (source_index, source_byte) in text.bytes().enumerate() { + let byte = normalize_ascii_byte(source_byte); + if is_python_ascii_space(byte) { + pending_space = !normalized.is_empty(); + continue; + } + if pending_space { + normalized.push(b' '); + offsets.push(source_index); + pending_space = false; + } + normalized.push(byte); + offsets.push(source_index); + } +} + +fn normalize_text_into( + text: &str, + normalized: &mut Vec, + offsets: &mut Vec, +) -> (bool, bool, usize) { + normalized.clear(); + offsets.clear(); + normalized.reserve(text.len()); + offsets.reserve(text.len()); + if text.is_ascii() { + normalize_ascii_text_into(text, normalized, offsets); + return (true, false, text.len()); + } + let mut pending_space = false; + let mut all_ascii = true; + let mut exceptional_ignorecase = false; + let mut source_len = 0usize; + for (source_index, source_character) in text.chars().enumerate() { + source_len = source_index + 1; + all_ascii &= source_character.is_ascii(); + exceptional_ignorecase |= matches!( + source_character, + '\u{0130}' | '\u{0131}' | '\u{017f}' | '\u{212a}' + ); + normalize_source_char( + source_character, + source_index, + normalized, + offsets, + &mut pending_space, + ); + } + (all_ascii, exceptional_ignorecase, source_len) +} + +#[inline] +fn normalized_contains(haystack: &[u8], needle: &[u8]) -> bool { + if needle.is_empty() || haystack.len() < needle.len() { + return false; + } + let mut cursor = 0usize; + while cursor + needle.len() <= haystack.len() { + if let Some(relative) = haystack[cursor..] + .windows(needle.len()) + .position(|window| window == needle) + { + let start = cursor + relative; + let end = start + needle.len(); + let left_ok = start == 0 || !is_token_byte(haystack[start - 1]); + let right_ok = end == haystack.len() || !is_token_byte(haystack[end]); + if left_ok && right_ok { + return true; + } + cursor = start + 1; + } else { + return false; + } + } + false +} + +fn normalized_context_window<'a>( + normalized: &'a [u8], + offsets: &[usize], + source_len: usize, + start: usize, + end: usize, + width: usize, +) -> &'a [u8] { + debug_assert_eq!(normalized.len(), offsets.len()); + debug_assert!(start <= end); + debug_assert!(end <= source_len); + + let left = start.saturating_sub(width); + let right = end.saturating_add(width).min(source_len); + let normalized_left = offsets.partition_point(|&source_index| source_index < left); + let normalized_right = offsets.partition_point(|&source_index| source_index < right); + let mut window = &normalized[normalized_left..normalized_right]; + + // A full-row normalization can carry a collapsed separator from before the + // source window to its first non-separator. Normalizing the source window in + // isolation drops that leading separator, so remove it from the borrowed + // slice as well. Trailing separators already map to the first source + // character after the window and are excluded by normalized_right. + if window.first() == Some(&b' ') { + window = &window[1..]; + } + window +} + +fn context_status( + normalized: &[u8], + offsets: &[usize], + source_len: usize, + start: usize, + end: usize, + rule: &ContextRule, +) -> u8 { + // 0=satisfied, 1=explicit_collision_exclusion, 2=context_clause_not_satisfied + let window = normalized_context_window( + normalized, + offsets, + source_len, + start, + end, + rule.window_chars, + ); + if rule + .reject_terms + .iter() + .any(|term| normalized_contains(window, term)) + { + return 1; + } + if rule + .accept_clauses + .iter() + .any(|clause| clause.iter().all(|term| normalized_contains(window, term))) + { + 0 + } else { + 2 + } +} + +#[inline(always)] +fn py_ignorecase_ascii_literal_eq(pattern: char, observed: char) -> bool { + let lower = pattern.to_ascii_lowercase(); + if observed.is_ascii() { + return observed.to_ascii_lowercase() == lower; + } + match lower { + 'i' => matches!(observed, 'İ' | 'ı'), + 's' => observed == 'ſ', + 'k' => observed == 'K', + _ => false, + } +} + +fn exact_matches(chars: &[char], spec: &ExactSpec) -> Vec<(usize, usize)> { + let width = spec.value.len(); + if width == 0 || width > chars.len() { + return Vec::new(); + } + let mut output = Vec::new(); + for start in 0..=chars.len() - width { + let end = start + width; + if start > 0 && is_exact_token_char(chars[start - 1], spec.mode) { + continue; + } + if end < chars.len() && is_exact_token_char(chars[end], spec.mode) { + continue; + } + let matched = spec.value.iter().enumerate().all(|(offset, expected)| { + let observed = chars[start + offset]; + match spec.mode { + ExactMode::CaseSensitive => observed == *expected, + ExactMode::PythonIgnoreCaseAsciiLiteral => { + py_ignorecase_ascii_literal_eq(*expected, observed) + } + } + }); + if matched { + output.push((start, end)); + } + } + output +} + +#[inline(always)] +fn exact_boundaries_match(text: &str, start: usize, end: usize, mode: ExactMode) -> bool { + let left_ok = start == 0 + || !is_exact_token_char( + text[..start] + .chars() + .next_back() + .expect("nonempty exact left boundary"), + mode, + ); + let right_ok = end == text.len() + || !is_exact_token_char( + text[end..] + .chars() + .next() + .expect("nonempty exact right boundary"), + mode, + ); + left_ok && right_ok +} + +impl ExactAutomaton { + fn append_matches(&self, text: &str, mode: ExactMode, output: &mut Vec) { + for matched in self.automaton.find_overlapping_iter(text.as_bytes()) { + let start = matched.start(); + let end = matched.end(); + if !exact_boundaries_match(text, start, end, mode) { + continue; + } + for &spec_index in &self.pattern_specs[matched.pattern().as_usize()] { + output.push(ExactByteMatch { + spec_index, + start, + end, + }); + } + } + } +} + +fn build_exact_automaton( + specs: &[ExactSpec], + mode: ExactMode, + ascii_case_insensitive: bool, +) -> PyResult> { + let mut patterns = Vec::::new(); + let mut pattern_specs = Vec::>::new(); + let mut pattern_ids = HashMap::::new(); + + for (spec_index, spec) in specs.iter().enumerate() { + if spec.mode != mode { + continue; + } + let value: String = spec.value.iter().collect(); + let pattern_key = if ascii_case_insensitive { + value.to_ascii_lowercase() + } else { + value.clone() + }; + if let Some(&pattern_id) = pattern_ids.get(&pattern_key) { + pattern_specs[pattern_id].push(spec_index); + } else { + let pattern_id = patterns.len(); + pattern_ids.insert(pattern_key, pattern_id); + patterns.push(value); + pattern_specs.push(vec![spec_index]); + } + } + + if patterns.is_empty() { + return Ok(None); + } + let automaton = build_auto_automaton(&patterns, ascii_case_insensitive)?; + Ok(Some(ExactAutomaton { + automaton, + pattern_specs, + })) +} + +impl ExactSpecs { + fn new(specs: Vec) -> PyResult { + let case_sensitive = build_exact_automaton(&specs, ExactMode::CaseSensitive, false)?; + let ignorecase_ascii = + build_exact_automaton(&specs, ExactMode::PythonIgnoreCaseAsciiLiteral, true)?; + Ok(Self { + specs, + case_sensitive, + ignorecase_ascii, + }) + } + + fn append_matching( + &self, + text: &str, + byte_matches: &mut Vec, + output: &mut Vec<(usize, usize, usize)>, + exceptional_chars: &mut Vec, + boundaries: &mut Vec, + text_is_ascii: bool, + exceptional_ignorecase: bool, + ) { + byte_matches.clear(); + output.clear(); + exceptional_chars.clear(); + boundaries.clear(); + if let Some(automaton) = &self.case_sensitive { + automaton.append_matches(text, ExactMode::CaseSensitive, byte_matches); + } + + if !exceptional_ignorecase { + if let Some(automaton) = &self.ignorecase_ascii { + automaton.append_matches( + text, + ExactMode::PythonIgnoreCaseAsciiLiteral, + byte_matches, + ); + } + } + + output.reserve(byte_matches.len()); + if text_is_ascii { + output.extend( + byte_matches + .iter() + .copied() + .map(|matched| (matched.spec_index, matched.start, matched.end)), + ); + } else if !byte_matches.is_empty() { + // Aho-Corasick reports UTF-8 byte offsets while the frozen Python + // surface reports code-point offsets. Build this map only for the + // uncommon non-ASCII row that actually produced an exact hit. + boundaries.extend(text.char_indices().map(|(offset, _)| offset)); + boundaries.push(text.len()); + output.extend(byte_matches.iter().copied().map(|matched| { + let start = boundaries + .binary_search(&matched.start) + .expect("exact match starts at a UTF-8 character boundary"); + let end = boundaries + .binary_search(&matched.end) + .expect("exact match ends at a UTF-8 character boundary"); + (matched.spec_index, start, end) + })); + } + + if exceptional_ignorecase { + // Python re.IGNORECASE gives ASCII I/S/K four additional Unicode + // equivalents (İ, ı, ſ, K). The common ASCII automaton cannot change + // byte width while matching, so retain the exact scalar fallback + // only for rows that contain one of those exceptional code points. + exceptional_chars.extend(text.chars()); + for (spec_index, spec) in self.specs.iter().enumerate() { + if spec.mode != ExactMode::PythonIgnoreCaseAsciiLiteral { + continue; + } + output.extend( + exact_matches(exceptional_chars, spec) + .into_iter() + .map(|(start, end)| (spec_index, start, end)), + ); + } + } + } + + #[cfg(test)] + fn matching(&self, text: &str) -> Vec<(usize, usize, usize)> { + let mut byte_matches = Vec::new(); + let mut output = Vec::new(); + let mut exceptional_chars = Vec::new(); + let mut boundaries = Vec::new(); + let mut normalized = Vec::new(); + let mut offsets = Vec::new(); + let (text_is_ascii, exceptional_ignorecase, _) = + normalize_text_into(text, &mut normalized, &mut offsets); + self.append_matching( + text, + &mut byte_matches, + &mut output, + &mut exceptional_chars, + &mut boundaries, + text_is_ascii, + exceptional_ignorecase, + ); + output + } +} + +fn scan_one( + automaton: &AhoCorasick, + pattern_consumers: &[Vec], + consumers: &[ConsumerSpec], + exact_specs: &ExactSpecs, + aliases: &[AliasSpec], + lexical: &[LexicalSpec], + contexts: &[ContextRule], + conjunctions: &[ConjunctionSpec], + text: &str, + observed_epoch: i64, + row_index: usize, + scratch: &mut ScanScratch, +) -> Option { + scratch.reset(conjunctions.len()); + let (text_is_ascii, exceptional_ignorecase, source_len) = + normalize_text_into(text, &mut scratch.normalized, &mut scratch.offsets); + let mut native_candidate = false; + if !scratch.normalized.is_empty() { + for matched in automaton.find_overlapping_iter(&scratch.normalized) { + let start = matched.start(); + let end = matched.end(); + let left_ok = start == 0 || !is_token_byte(scratch.normalized[start - 1]); + let right_ok = + end == scratch.normalized.len() || !is_token_byte(scratch.normalized[end]); + if !left_ok || !right_ok { + continue; + } + native_candidate = true; + let original_start = scratch.offsets[start]; + let original_end = scratch.offsets[end - 1] + 1; + for &consumer_id in &pattern_consumers[matched.pattern().as_usize()] { + let spec = &consumers[consumer_id]; + match spec.kind { + ALIAS => { + scratch + .alias_candidates + .push((spec.a, original_start, original_end)); + } + LEXICAL => { + scratch + .lexical_candidates + .push((spec.a, original_start, original_end)); + } + CONJUNCTION => { + if conjunctions + .get(spec.a) + .is_some_and(|rule| rule.contains_phrase(spec.b, spec.c)) + { + scratch.conjunction_marks.mark(spec.a, spec.b); + } + } + _ => unreachable!("validated consumer kind"), + } + } + } + } + + let mut exact_candidate = false; + exact_specs.append_matching( + text, + &mut scratch.exact_byte_matches, + &mut scratch.exact_matches, + &mut scratch.chars, + &mut scratch.exact_boundaries, + text_is_ascii, + exceptional_ignorecase, + ); + for &(spec_index, start, end) in &scratch.exact_matches { + exact_candidate = true; + let spec = &exact_specs.specs[spec_index]; + match spec.kind { + ALIAS => scratch.alias_candidates.push((spec.index, start, end)), + LEXICAL => scratch.lexical_candidates.push((spec.index, start, end)), + CONJUNCTION => { + scratch.conjunction_marks.mark(spec.index, 0); + } + _ => unreachable!("validated exact kind"), + } + } + + if scratch.alias_candidates.is_empty() + && scratch.lexical_candidates.is_empty() + && scratch.conjunction_marks.is_empty() + { + return None; + } + + let mut accepted_nonsemantic = false; + let mut accepted_evidence = false; + let mut unresolved_exposure = false; + let mut unresolved_event = false; + let mut supported_mask: u8 = 0; + let mut observation_count = 0usize; + + // Group aliases by (canonical_id, start) without a per-row hash table. The + // longest end in each group wins, while distinct aliases that tie remain + // distinct observations. Exact duplicate candidates collapse here. + scratch.alias_candidates.sort_unstable_by_key( + |&(alias_index, start, end)| { + (aliases[alias_index].canonical_group, start, end, alias_index) + }, + ); + scratch.alias_candidates.dedup(); + let mut group_start = 0usize; + while group_start < scratch.alias_candidates.len() { + let (first_alias, first_start, _) = scratch.alias_candidates[group_start]; + let canonical_group = aliases[first_alias].canonical_group; + let mut group_end = group_start + 1; + let mut longest_end = scratch.alias_candidates[group_start].2; + while group_end < scratch.alias_candidates.len() { + let (alias_index, start, end) = scratch.alias_candidates[group_end]; + if aliases[alias_index].canonical_group != canonical_group || start != first_start { + break; + } + longest_end = longest_end.max(end); + group_end += 1; + } + + for candidate_index in group_start..group_end { + let (alias_index, start, end) = scratch.alias_candidates[candidate_index]; + if end < longest_end { + continue; + } + let spec = &aliases[alias_index]; + + // The frozen compact contract counts a retained observation even + // when its temporal or context checks later reject it. + observation_count += 1; + let active = observed_epoch >= spec.valid_from_epoch + && spec + .valid_to_epoch + .map(|until| observed_epoch < until) + .unwrap_or(true); + if !active { + continue; + } + + let context_code = if let Some(rule_index) = spec.context_rule { + context_status( + &scratch.normalized, + &scratch.offsets, + source_len, + start, + end, + &contexts[rule_index], + ) + } else { + 0 + }; + if context_code != 0 { + // Frozen exposure unresolved filter excludes explicit collision and + // context-clause failures. Missing context cannot occur because the + // validated config rejects unknown rules during construction. + continue; + } + + accepted_nonsemantic = true; + if spec.family != FAMILY_NONE && (spec.entity_kind == 0 || spec.entity_kind == 1) { + accepted_evidence = true; + supported_mask |= 1u8 << (spec.family as u8); + } else if spec.unresolved_tag { + unresolved_exposure = true; + } + } + group_start = group_end; + } + + // Observation identity for lexical matches is (rule, start, end), so the + // sorted sparse candidates replace the former observation HashSet exactly. + scratch.lexical_candidates.sort_unstable(); + scratch.lexical_candidates.dedup(); + for candidate_index in 0..scratch.lexical_candidates.len() { + let (lexical_index, start, end) = scratch.lexical_candidates[candidate_index]; + let spec = &lexical[lexical_index]; + observation_count += 1; + + let context_ok = if spec.context_terms.is_empty() { + true + } else { + let window = normalized_context_window( + &scratch.normalized, + &scratch.offsets, + source_len, + start, + end, + spec.context_window_chars, + ); + spec.context_terms + .iter() + .any(|term| normalized_contains(window, term)) + }; + + if spec.is_event { + if spec.market_wide_authorized { + accepted_evidence = true; + supported_mask |= 1u8 << FAMILY_MARKET_WIDE; + } else { + if spec.unresolved_tag { + unresolved_exposure = true; + } + if spec.has_event_family { + unresolved_event = true; + } + } + } else { + if context_ok { + accepted_nonsemantic = true; + } + if spec.market_wide_authorized && context_ok { + accepted_evidence = true; + supported_mask |= 1u8 << FAMILY_MARKET_WIDE; + } else if spec.unresolved_tag { + // Matches frozen generic lexical unresolved branch, including a + // lexical context failure, whose evidence reason is the generic + // identity-resolved-without-path code. + unresolved_exposure = true; + } + } + } + + for touched_index in 0..scratch.conjunction_marks.touched.len() { + let rule_index = scratch.conjunction_marks.touched[touched_index]; + if scratch.conjunction_marks.is_complete(rule_index) { + // Only one conjunction observation exists per configured rule. + observation_count += 1; + accepted_evidence = true; + supported_mask |= 1u8 << FAMILY_MARKET_WIDE; + } + } + + if observation_count == 0 { + return None; + } + let article_status = if accepted_evidence || accepted_nonsemantic || unresolved_event { + ARTICLE_ACCEPTED + } else { + ARTICLE_REJECTED + }; + let completed_elsewhere = supported_mask != 0; + let exposure_status = |family: i8| -> u8 { + if (supported_mask & (1u8 << (family as u8))) != 0 { + EXP_SUPPORTED + } else if family != FAMILY_MARKET_WIDE && !completed_elsewhere && unresolved_exposure { + EXP_PLAUSIBLE + } else { + EXP_NOT_SUPPORTED + } + }; + + Some(( + row_index, + article_status, + exposure_status(FAMILY_SP500), + exposure_status(FAMILY_NASDAQ100), + exposure_status(FAMILY_RUSSELL2000), + exposure_status(FAMILY_MARKET_WIDE), + observation_count, + native_candidate, + exact_candidate, + )) +} + #[pyclass(frozen, module = "range_news._native")] -struct NativeCompactFactScanner { - automaton: AhoCorasick, - pattern_consumers: Vec>, - consumers: Vec, - exact_specs: ExactSpecs, - aliases: Vec, - lexical: Vec, - contexts: Vec, - conjunctions: Vec, -} - -#[pymethods] -impl NativeCompactFactScanner { - #[new] - #[pyo3(signature = ( - patterns, - pattern_consumers, - consumers, - exact_specs, - alias_specs, - lexical_specs, - context_rules, - conjunction_specs - ))] - fn new( - patterns: Vec, - pattern_consumers: Vec>, - consumers: Vec<(u8, usize, usize, usize)>, - exact_specs: Vec<(String, String, u8, usize)>, - alias_specs: Vec<(usize, i64, Option, i8, u8, bool, Option)>, - lexical_specs: Vec<(usize, Vec, bool, bool, bool, bool)>, - context_rules: Vec<(usize, Vec>, Vec)>, - conjunction_specs: Vec<(usize, usize, usize, usize)>, - ) -> PyResult { - if patterns.len() != pattern_consumers.len() { - return Err(PyValueError::new_err("pattern_consumers length mismatch")); - } - let automaton = build_automaton(&patterns)?; - let consumers = consumers - .into_iter() - .map(|(kind, a, b, c)| ConsumerSpec { kind, a, b, c }) - .collect(); - let mut parsed_exact = Vec::with_capacity(exact_specs.len()); - for (value, mode, kind, index) in exact_specs { - if value.is_empty() { - return Err(PyValueError::new_err("exact value must be non-empty")); - } - let mode = match mode.as_str() { - "case_sensitive" => ExactMode::CaseSensitive, - "python_ignorecase_ascii_literal" => { - if !value.is_ascii() { - return Err(PyValueError::new_err("ignorecase literal must be ASCII")); - } - ExactMode::PythonIgnoreCaseAsciiLiteral - } - _ => return Err(PyValueError::new_err(format!("unknown exact mode: {mode}"))), - }; - parsed_exact.push(ExactSpec { - value: value.chars().collect(), - mode, - kind, - index, - }); - } - let aliases = alias_specs - .into_iter() - .map( - |( - canonical_group, - valid_from_epoch, - valid_to_epoch, - family, - entity_kind, - unresolved_tag, - context_rule, - )| AliasSpec { - canonical_group, - valid_from_epoch, - valid_to_epoch, - family, - entity_kind, - unresolved_tag, - context_rule, - }, - ) - .collect(); - let lexical = lexical_specs - .into_iter() - .map( - |( - context_window_chars, - terms, - is_event, - market_wide_authorized, - unresolved_tag, - has_event_family, - )| LexicalSpec { - context_window_chars, - context_terms: terms.into_iter().map(String::into_bytes).collect(), - is_event, - market_wide_authorized, - unresolved_tag, - has_event_family, - }, - ) - .collect(); - let contexts = context_rules - .into_iter() - .map(|(window_chars, clauses, rejects)| ContextRule { - window_chars, - accept_clauses: clauses - .into_iter() - .map(|clause| clause.into_iter().map(String::into_bytes).collect()) - .collect(), - reject_terms: rejects.into_iter().map(String::into_bytes).collect(), - }) - .collect(); - let conjunctions = conjunction_specs - .into_iter() - .map( - |( - actor_phrase_count, - action_phrase_count, - object_phrase_count, - scope_phrase_count, - )| ConjunctionSpec { - actor_phrase_count, - action_phrase_count, - object_phrase_count, - scope_phrase_count, - }, - ) - .collect(); - let exact_specs = ExactSpecs::new(parsed_exact)?; - Ok(Self { - automaton, - pattern_consumers, - consumers, - exact_specs, - aliases, - lexical, - contexts, - conjunctions, - }) - } - - #[pyo3(signature = (texts, epochs, workers=4))] - fn screen_facts_batch( - &self, - py: Python<'_>, - texts: Vec, - epochs: Vec, - workers: usize, - ) -> PyResult> { - if texts.len() != epochs.len() { - return Err(PyValueError::new_err("texts/epochs length mismatch")); - } - if workers == 0 { - return Err(PyValueError::new_err("workers must be >= 1")); - } - let pool = ThreadPoolBuilder::new() - .num_threads(workers) - .thread_name(|index| format!("news-relevance-worker-{index}")) - .build() - .map_err(|error| PyValueError::new_err(error.to_string()))?; - let automaton = &self.automaton; - let pattern_consumers = &self.pattern_consumers; - let consumers = &self.consumers; - let exact_specs = &self.exact_specs; - let aliases = &self.aliases; - let lexical = &self.lexical; - let contexts = &self.contexts; - let conjunctions = &self.conjunctions; - Ok(py.detach(move || { - pool.install(|| { - texts - .par_iter() - .zip(epochs.par_iter()) - .enumerate() - .map_init(ScanScratch::default, |scratch, (index, (text, epoch))| { - scan_one( - automaton, - pattern_consumers, - consumers, - exact_specs, - aliases, - lexical, - contexts, - conjunctions, - text, - *epoch, - index, - scratch, - ) - }) - .filter_map(|row| row) - .collect() - }) - })) - } -} - -#[cfg(test)] -mod exact_automata_tests { - use super::*; - - fn spec(value: &str, mode: ExactMode, index: usize) -> ExactSpec { - ExactSpec { - value: value.chars().collect(), - mode, - kind: ALIAS, - index, - } - } - - #[test] - fn exact_automata_equal_scalar_reference_across_boundaries_and_unicode() { - let specs = vec![ - spec("UPRO", ExactMode::CaseSensitive, 0), - spec("QQQ", ExactMode::PythonIgnoreCaseAsciiLiteral, 1), - spec("iShares", ExactMode::PythonIgnoreCaseAsciiLiteral, 2), - spec("PowerShares", ExactMode::PythonIgnoreCaseAsciiLiteral, 3), - // Duplicate literals must still fan out to every configured consumer. - spec("SEC", ExactMode::CaseSensitive, 4), - spec("SEC", ExactMode::CaseSensitive, 5), - ]; - let matcher = ExactSpecs::new(specs).expect("valid exact automata"); - let texts = [ - "UPRO upro QQQ qqq xQQQ QQQx SEC", - "é UPRO αQQQ αSEC", - "İShares ıSHARES iShares Powerſhares", - "İQQQ QQQİ ſQQQ QQQK", - "no exact literals here", - ]; - - for text in texts { - let chars: Vec = text.chars().collect(); - let mut expected = Vec::new(); - for (spec_index, spec) in matcher.specs.iter().enumerate() { - expected.extend( - exact_matches(&chars, spec) - .into_iter() - .map(|(start, end)| (spec_index, start, end)), - ); - } - expected.sort_unstable(); - - let mut observed = matcher.matching(text); - observed.sort_unstable(); - assert_eq!(observed, expected, "text={text:?}"); - assert_eq!(!observed.is_empty(), !expected.is_empty()); - } - } - - #[test] - fn python_ignorecase_exceptions_remain_token_boundaries() { - let matcher = ExactSpecs::new(vec![spec( - "QQQ", - ExactMode::PythonIgnoreCaseAsciiLiteral, - 0, - )]) - .expect("valid exact automaton"); - for text in ["İQQQ", "QQQı", "ſQQQ", "QQQK"] { - assert!(matcher.matching(text).is_empty(), "text={text:?}"); - } - } - - #[test] - fn direct_text_normalization_matches_character_slice_reference() { - let texts = [ - "ASCII -- punctuation\tand spaces", - "Café — ‘quoted’ / fullwidth", - "Straße İ ı ſ K fi", - "\u{001c}leading\u{00a0}and\u{2029}trailing\n", - "", - ]; - for text in texts { - let chars: Vec = text.chars().collect(); - let mut expected = Vec::new(); - let mut expected_offsets = Vec::new(); - normalize_chars_into(&chars, 0, &mut expected, &mut expected_offsets); - - let mut observed = Vec::new(); - let mut observed_offsets = Vec::new(); - let (all_ascii, exceptional_ignorecase) = - normalize_text_into(text, &mut observed, &mut observed_offsets); - assert_eq!(observed, expected, "text={text:?}"); - assert_eq!(observed_offsets, expected_offsets, "text={text:?}"); - assert_eq!( - all_ascii, - chars.iter().all(|character| character.is_ascii()) - ); - assert_eq!( - exceptional_ignorecase, - chars.iter().any(|character| matches!( - character, - '\u{0130}' | '\u{0131}' | '\u{017f}' | '\u{212a}' - )), - "text={text:?}" - ); - } - } - - fn assert_ascii_normalization_matches_character_reference(bytes: &[u8]) { - let text = std::str::from_utf8(bytes).expect("ASCII bytes are valid UTF-8"); - let chars: Vec = text.chars().collect(); - let mut expected = Vec::new(); - let mut expected_offsets = Vec::new(); - normalize_chars_into(&chars, 0, &mut expected, &mut expected_offsets); - - let mut observed = Vec::new(); - let mut observed_offsets = Vec::new(); - let flags = normalize_text_into(text, &mut observed, &mut observed_offsets); - assert_eq!(flags, (true, false), "bytes={bytes:?}"); - assert_eq!(observed, expected, "bytes={bytes:?}"); - assert_eq!(observed_offsets, expected_offsets, "bytes={bytes:?}"); - } - - #[test] - fn ascii_byte_path_matches_character_reference_exhaustively() { - assert_ascii_normalization_matches_character_reference(b""); - for first in 0u8..=0x7f { - assert_ascii_normalization_matches_character_reference(&[first]); - assert_ascii_normalization_matches_character_reference(&[b'x', first, b'y']); - for second in 0u8..=0x7f { - assert_ascii_normalization_matches_character_reference(&[first, second]); - assert_ascii_normalization_matches_character_reference(&[ - b'x', first, second, b'y', - ]); - } - } - } - - #[test] - fn ascii_byte_path_preserves_collapse_and_control_semantics() { - let tricky = [ - b' ', b'-', b'\t', b'A', b'`', b'B', b'.', b'/', b' ', 0x00, b'\\', 0x7f, b' ', b'\n', - ]; - assert_ascii_normalization_matches_character_reference(&tricky); - - let mut normalized = Vec::new(); - let mut offsets = Vec::new(); - let text = std::str::from_utf8(&tricky).expect("ASCII bytes are valid UTF-8"); - assert_eq!( - normalize_text_into(text, &mut normalized, &mut offsets), - (true, false) - ); - assert_eq!(normalized, b"a'b \0\\\x7f"); - assert_eq!(offsets, [3, 4, 5, 9, 9, 10, 11]); - - let python_ascii_whitespace = [ - b' ', b'\t', b'\n', 0x0b, 0x0c, b'\r', 0x1c, 0x1d, 0x1e, 0x1f, - ]; - let separators = [ - b'-', b'.', b',', b':', b';', b'(', b')', b'[', b']', b'{', b'}', b'/', - ]; - let mut runs = vec![b'A']; - runs.extend_from_slice(&python_ascii_whitespace); - runs.extend_from_slice(&separators); - runs.push(b'Z'); - assert_ascii_normalization_matches_character_reference(&runs); - normalize_text_into( - std::str::from_utf8(&runs).expect("ASCII bytes are valid UTF-8"), - &mut normalized, - &mut offsets, - ); - assert_eq!(normalized, b"a z"); - assert_eq!(offsets, [0, runs.len() - 1, runs.len() - 1]); - } - - #[test] - fn reused_scan_scratch_does_not_leak_candidates_between_rows() { - let automaton = build_automaton(&["alpha".to_string()]).expect("valid automaton"); - let pattern_consumers = vec![vec![0]]; - let consumers = vec![ConsumerSpec { - kind: ALIAS, - a: 0, - b: 0, - c: 0, - }]; - let exact_specs = ExactSpecs::new(Vec::new()).expect("empty exact specs are valid"); - let aliases = vec![AliasSpec { - canonical_group: 0, - context_rule: None, - valid_from_epoch: 0, - valid_to_epoch: None, - family: FAMILY_SP500, - entity_kind: 0, - unresolved_tag: false, - }]; - let mut scratch = ScanScratch::default(); - let scan = |text: &str, row_index: usize, scratch: &mut ScanScratch| { - scan_one( - &automaton, - &pattern_consumers, - &consumers, - &exact_specs, - &aliases, - &[], - &[], - &[], - text, - 1, - row_index, - scratch, - ) - }; - - let first = scan("Alpha rises", 0, &mut scratch).expect("first row matches"); - assert_eq!(first.0, 0); - assert_eq!(first.6, 1); - assert!(scan("nothing relevant", 1, &mut scratch).is_none()); - let third = scan("alpha again", 2, &mut scratch).expect("third row matches"); - assert_eq!(third.0, 2); - assert_eq!(third.6, 1); - } -} +struct NativeCompactFactScanner { + automaton: AhoCorasick, + pattern_consumers: Vec>, + consumers: Vec, + exact_specs: ExactSpecs, + aliases: Vec, + lexical: Vec, + contexts: Vec, + conjunctions: Vec, +} + +#[pymethods] +impl NativeCompactFactScanner { + #[new] + #[pyo3(signature = ( + patterns, + pattern_consumers, + consumers, + exact_specs, + alias_specs, + lexical_specs, + context_rules, + conjunction_specs + ))] + fn new( + patterns: Vec, + pattern_consumers: Vec>, + consumers: Vec<(u8, usize, usize, usize)>, + exact_specs: Vec<(String, String, u8, usize)>, + alias_specs: Vec<(usize, i64, Option, i8, u8, bool, Option)>, + lexical_specs: Vec<(usize, Vec, bool, bool, bool, bool)>, + context_rules: Vec<(usize, Vec>, Vec)>, + conjunction_specs: Vec<(usize, usize, usize, usize)>, + ) -> PyResult { + if patterns.len() != pattern_consumers.len() { + return Err(PyValueError::new_err("pattern_consumers length mismatch")); + } + let automaton = build_automaton(&patterns)?; + let consumers = consumers + .into_iter() + .map(|(kind, a, b, c)| ConsumerSpec { kind, a, b, c }) + .collect(); + let mut parsed_exact = Vec::with_capacity(exact_specs.len()); + for (value, mode, kind, index) in exact_specs { + if value.is_empty() { + return Err(PyValueError::new_err("exact value must be non-empty")); + } + let mode = match mode.as_str() { + "case_sensitive" => ExactMode::CaseSensitive, + "python_ignorecase_ascii_literal" => { + if !value.is_ascii() { + return Err(PyValueError::new_err("ignorecase literal must be ASCII")); + } + ExactMode::PythonIgnoreCaseAsciiLiteral + } + _ => return Err(PyValueError::new_err(format!("unknown exact mode: {mode}"))), + }; + parsed_exact.push(ExactSpec { + value: value.chars().collect(), + mode, + kind, + index, + }); + } + let aliases = alias_specs + .into_iter() + .map( + |( + canonical_group, + valid_from_epoch, + valid_to_epoch, + family, + entity_kind, + unresolved_tag, + context_rule, + )| AliasSpec { + canonical_group, + valid_from_epoch, + valid_to_epoch, + family, + entity_kind, + unresolved_tag, + context_rule, + }, + ) + .collect(); + let lexical = lexical_specs + .into_iter() + .map( + |( + context_window_chars, + terms, + is_event, + market_wide_authorized, + unresolved_tag, + has_event_family, + )| LexicalSpec { + context_window_chars, + context_terms: terms.into_iter().map(String::into_bytes).collect(), + is_event, + market_wide_authorized, + unresolved_tag, + has_event_family, + }, + ) + .collect(); + let contexts = context_rules + .into_iter() + .map(|(window_chars, clauses, rejects)| ContextRule { + window_chars, + accept_clauses: clauses + .into_iter() + .map(|clause| clause.into_iter().map(String::into_bytes).collect()) + .collect(), + reject_terms: rejects.into_iter().map(String::into_bytes).collect(), + }) + .collect(); + let conjunctions = conjunction_specs + .into_iter() + .map( + |( + actor_phrase_count, + action_phrase_count, + object_phrase_count, + scope_phrase_count, + )| ConjunctionSpec { + actor_phrase_count, + action_phrase_count, + object_phrase_count, + scope_phrase_count, + }, + ) + .collect(); + let exact_specs = ExactSpecs::new(parsed_exact)?; + Ok(Self { + automaton, + pattern_consumers, + consumers, + exact_specs, + aliases, + lexical, + contexts, + conjunctions, + }) + } + + #[pyo3(signature = (texts, epochs, workers=4))] + fn screen_facts_batch( + &self, + py: Python<'_>, + texts: Vec, + epochs: Vec, + workers: usize, + ) -> PyResult> { + if texts.len() != epochs.len() { + return Err(PyValueError::new_err("texts/epochs length mismatch")); + } + if workers == 0 { + return Err(PyValueError::new_err("workers must be >= 1")); + } + let pool = ThreadPoolBuilder::new() + .num_threads(workers) + .thread_name(|index| format!("news-relevance-worker-{index}")) + .build() + .map_err(|error| PyValueError::new_err(error.to_string()))?; + let automaton = &self.automaton; + let pattern_consumers = &self.pattern_consumers; + let consumers = &self.consumers; + let exact_specs = &self.exact_specs; + let aliases = &self.aliases; + let lexical = &self.lexical; + let contexts = &self.contexts; + let conjunctions = &self.conjunctions; + Ok(py.detach(move || { + pool.install(|| { + texts + .par_iter() + .zip(epochs.par_iter()) + .enumerate() + .map_init(ScanScratch::default, |scratch, (index, (text, epoch))| { + scan_one( + automaton, + pattern_consumers, + consumers, + exact_specs, + aliases, + lexical, + contexts, + conjunctions, + text, + *epoch, + index, + scratch, + ) + }) + .filter_map(|row| row) + .collect() + }) + })) + } +} + +#[cfg(test)] +mod matcher_tests { + use super::*; + + fn spec(value: &str, mode: ExactMode, index: usize) -> ExactSpec { + ExactSpec { + value: value.chars().collect(), + mode, + kind: ALIAS, + index, + } + } + + fn synthetic_patterns(count: usize) -> Vec { + (0..count) + .map(|index| format!("synthetic pattern {index:04}")) + .collect() + } + + fn overlapping_matches(automaton: &AhoCorasick, text: &str) -> Vec<(usize, usize, usize)> { + let mut matches: Vec<_> = automaton + .find_overlapping_iter(text.as_bytes()) + .map(|matched| { + ( + matched.pattern().as_usize(), + matched.start(), + matched.end(), + ) + }) + .collect(); + matches.sort_unstable(); + matches + } + + #[test] + fn default_sized_normalized_taxonomy_selects_bounded_dfa() { + // The maintained taxonomy currently has 226 normalized patterns, so + // this sits above aho-corasick's automatic DFA threshold while still + // remaining comfortably inside RANGE's explicit safety bounds. + let patterns = synthetic_patterns(226); + assert!(normalized_dfa_eligible(&patterns)); + let automaton = build_automaton(&patterns).expect("bounded DFA builds"); + assert_eq!(automaton.kind(), AhoCorasickKind::DFA); + assert!(automaton.memory_usage() <= NORMALIZED_DFA_MAX_HEAP_BYTES); + } + + #[test] + fn large_custom_taxonomy_falls_back_to_ordinary_auto_selection() { + let patterns = synthetic_patterns(NORMALIZED_DFA_MAX_PATTERNS + 1); + assert!(!normalized_dfa_eligible(&patterns)); + let selected = build_automaton(&patterns).expect("fallback automaton builds"); + let automatic = build_auto_automaton(&patterns, false).expect("auto automaton builds"); + assert_eq!(selected.kind(), automatic.kind()); + assert_ne!(selected.kind(), AhoCorasickKind::DFA); + } + + #[test] + fn bounded_dfa_preserves_standard_overlapping_matches() { + let mut patterns = synthetic_patterns(101); + patterns.extend([ + "equity market".to_string(), + "equity market selloff".to_string(), + "market selloff".to_string(), + "capital rule".to_string(), + "rule".to_string(), + ]); + let text = "a capital rule preceded an equity market selloff"; + + let selected = build_automaton(&patterns).expect("bounded DFA builds"); + let automatic = build_auto_automaton(&patterns, false).expect("auto automaton builds"); + assert_eq!(selected.kind(), AhoCorasickKind::DFA); + assert_ne!(automatic.kind(), AhoCorasickKind::DFA); + assert_eq!(selected.match_kind(), MatchKind::Standard); + assert_eq!( + overlapping_matches(&selected, text), + overlapping_matches(&automatic, text) + ); + } + + #[test] + fn exact_automata_equal_scalar_reference_across_boundaries_and_unicode() { + let specs = vec![ + spec("UPRO", ExactMode::CaseSensitive, 0), + spec("QQQ", ExactMode::PythonIgnoreCaseAsciiLiteral, 1), + spec("iShares", ExactMode::PythonIgnoreCaseAsciiLiteral, 2), + spec("PowerShares", ExactMode::PythonIgnoreCaseAsciiLiteral, 3), + // Duplicate literals must still fan out to every configured consumer. + spec("SEC", ExactMode::CaseSensitive, 4), + spec("SEC", ExactMode::CaseSensitive, 5), + ]; + let matcher = ExactSpecs::new(specs).expect("valid exact automata"); + let texts = [ + "UPRO upro QQQ qqq xQQQ QQQx SEC", + "é UPRO αQQQ αSEC", + "İShares ıSHARES iShares Powerſhares", + "İQQQ QQQİ ſQQQ QQQK", + "no exact literals here", + ]; + + for text in texts { + let chars: Vec = text.chars().collect(); + let mut expected = Vec::new(); + for (spec_index, spec) in matcher.specs.iter().enumerate() { + expected.extend( + exact_matches(&chars, spec) + .into_iter() + .map(|(start, end)| (spec_index, start, end)), + ); + } + expected.sort_unstable(); + + let mut observed = matcher.matching(text); + observed.sort_unstable(); + assert_eq!(observed, expected, "text={text:?}"); + assert_eq!(!observed.is_empty(), !expected.is_empty()); + } + } + + #[test] + fn python_ignorecase_exceptions_remain_token_boundaries() { + let matcher = ExactSpecs::new(vec![spec( + "QQQ", + ExactMode::PythonIgnoreCaseAsciiLiteral, + 0, + )]) + .expect("valid exact automaton"); + for text in ["İQQQ", "QQQı", "ſQQQ", "QQQK"] { + assert!(matcher.matching(text).is_empty(), "text={text:?}"); + } + } + + #[test] + fn direct_text_normalization_matches_character_slice_reference() { + let texts = [ + "ASCII -- punctuation\tand spaces", + "Café — ‘quoted’ / fullwidth", + "Straße İ ı ſ K fi", + "\u{001c}leading\u{00a0}and\u{2029}trailing\n", + "", + ]; + for text in texts { + let chars: Vec = text.chars().collect(); + let mut expected = Vec::new(); + let mut expected_offsets = Vec::new(); + normalize_chars_into(&chars, 0, &mut expected, &mut expected_offsets); + + let mut observed = Vec::new(); + let mut observed_offsets = Vec::new(); + let (all_ascii, exceptional_ignorecase, source_len) = + normalize_text_into(text, &mut observed, &mut observed_offsets); + assert_eq!(observed, expected, "text={text:?}"); + assert_eq!(observed_offsets, expected_offsets, "text={text:?}"); + assert_eq!(source_len, chars.len(), "text={text:?}"); + assert_eq!( + all_ascii, + chars.iter().all(|character| character.is_ascii()) + ); + assert_eq!( + exceptional_ignorecase, + chars.iter().any(|character| matches!( + character, + '\u{0130}' | '\u{0131}' | '\u{017f}' | '\u{212a}' + )), + "text={text:?}" + ); + } + } + + fn assert_ascii_normalization_matches_character_reference(bytes: &[u8]) { + let text = std::str::from_utf8(bytes).expect("ASCII bytes are valid UTF-8"); + let chars: Vec = text.chars().collect(); + let mut expected = Vec::new(); + let mut expected_offsets = Vec::new(); + normalize_chars_into(&chars, 0, &mut expected, &mut expected_offsets); + + let mut observed = Vec::new(); + let mut observed_offsets = Vec::new(); + let flags = normalize_text_into(text, &mut observed, &mut observed_offsets); + assert_eq!(flags, (true, false, bytes.len()), "bytes={bytes:?}"); + assert_eq!(observed, expected, "bytes={bytes:?}"); + assert_eq!(observed_offsets, expected_offsets, "bytes={bytes:?}"); + } + + #[test] + fn ascii_byte_path_matches_character_reference_exhaustively() { + assert_ascii_normalization_matches_character_reference(b""); + for first in 0u8..=0x7f { + assert_ascii_normalization_matches_character_reference(&[first]); + assert_ascii_normalization_matches_character_reference(&[b'x', first, b'y']); + for second in 0u8..=0x7f { + assert_ascii_normalization_matches_character_reference(&[first, second]); + assert_ascii_normalization_matches_character_reference(&[ + b'x', first, second, b'y', + ]); + } + } + } + + #[test] + fn ascii_byte_path_preserves_collapse_and_control_semantics() { + let tricky = [ + b' ', b'-', b'\t', b'A', b'`', b'B', b'.', b'/', b' ', 0x00, b'\\', 0x7f, b' ', b'\n', + ]; + assert_ascii_normalization_matches_character_reference(&tricky); + + let mut normalized = Vec::new(); + let mut offsets = Vec::new(); + let text = std::str::from_utf8(&tricky).expect("ASCII bytes are valid UTF-8"); + assert_eq!( + normalize_text_into(text, &mut normalized, &mut offsets), + (true, false, tricky.len()) + ); + assert_eq!(normalized, b"a'b \0\\\x7f"); + assert_eq!(offsets, [3, 4, 5, 9, 9, 10, 11]); + + let python_ascii_whitespace = [ + b' ', b'\t', b'\n', 0x0b, 0x0c, b'\r', 0x1c, 0x1d, 0x1e, 0x1f, + ]; + let separators = [ + b'-', b'.', b',', b':', b';', b'(', b')', b'[', b']', b'{', b'}', b'/', + ]; + let mut runs = vec![b'A']; + runs.extend_from_slice(&python_ascii_whitespace); + runs.extend_from_slice(&separators); + runs.push(b'Z'); + assert_ascii_normalization_matches_character_reference(&runs); + normalize_text_into( + std::str::from_utf8(&runs).expect("ASCII bytes are valid UTF-8"), + &mut normalized, + &mut offsets, + ); + assert_eq!(normalized, b"a z"); + assert_eq!(offsets, [0, runs.len() - 1, runs.len() - 1]); + } + + #[test] + fn borrowed_context_windows_match_isolated_normalization_exhaustively() { + let texts = [ + "", + "a", + " --Federal\tReserve...cut rates ", + "prefix---GDP\u{00a0}growth/suffix", + "Café — ‘SEC’ / filing", + "İ ı ſ K fi — SPY ETF", + "x\u{001c}\u{001d}market\u{2029}structure\n", + ]; + + for text in texts { + let chars: Vec = text.chars().collect(); + let mut normalized = Vec::new(); + let mut offsets = Vec::new(); + let (_, _, source_len) = normalize_text_into(text, &mut normalized, &mut offsets); + assert_eq!(source_len, chars.len()); + + for start in 0..=source_len { + for end in start..=source_len { + for width in [0, 1, 2, 4, source_len.saturating_add(2)] { + let left = start.saturating_sub(width); + let right = end.saturating_add(width).min(source_len); + let mut expected = Vec::new(); + let mut expected_offsets = Vec::new(); + normalize_chars_into( + &chars[left..right], + left, + &mut expected, + &mut expected_offsets, + ); + let observed = normalized_context_window( + &normalized, + &offsets, + source_len, + start, + end, + width, + ); + assert_eq!( + observed, expected, + "text={text:?}, start={start}, end={end}, width={width}" + ); + } + } + } + } + } + + #[test] + fn borrowed_context_windows_preserve_phrase_boundaries() { + let text = "prefederal -- Federal Reserve; GDP-growth; SECond SEC filing; SPY ETF"; + let chars: Vec = text.chars().collect(); + let mut normalized = Vec::new(); + let mut offsets = Vec::new(); + let (_, _, source_len) = normalize_text_into(text, &mut normalized, &mut offsets); + let terms: [&[u8]; 7] = [ + b"fed", + b"federal reserve", + b"gdp", + b"growth", + b"sec", + b"spy", + b"etf", + ]; + + for start in 0..=source_len { + for end in start..=source_len { + for width in [0, 1, 8, source_len.saturating_add(1)] { + let left = start.saturating_sub(width); + let right = end.saturating_add(width).min(source_len); + let mut expected = Vec::new(); + let mut expected_offsets = Vec::new(); + normalize_chars_into( + &chars[left..right], + left, + &mut expected, + &mut expected_offsets, + ); + let observed = normalized_context_window( + &normalized, + &offsets, + source_len, + start, + end, + width, + ); + for term in terms { + assert_eq!( + normalized_contains(observed, term), + normalized_contains(&expected, term), + "term={term:?}, start={start}, end={end}, width={width}" + ); + } + } + } + } + } + + #[test] + fn custom_spy_context_keeps_reject_precedence() { + let automaton = + build_automaton(&["never matches".to_string()]).expect("valid normalized automaton"); + let exact_specs = ExactSpecs::new(vec![ExactSpec { + value: "SPY".chars().collect(), + mode: ExactMode::CaseSensitive, + kind: ALIAS, + index: 0, + }]) + .expect("valid exact alias"); + let aliases = vec![AliasSpec { + canonical_group: 0, + context_rule: Some(0), + valid_from_epoch: 0, + valid_to_epoch: None, + family: FAMILY_SP500, + entity_kind: 0, + unresolved_tag: false, + }]; + let contexts = vec![ContextRule { + window_chars: 128, + accept_clauses: vec![ + vec![b"etf".to_vec()], + vec![b"fund".to_vec(), b"s&p 500".to_vec()], + ], + reject_terms: vec![b"spy plane".to_vec(), b"espionage".to_vec()], + }]; + let mut scratch = ScanScratch::default(); + let scan = |text: &str, row_index: usize, scratch: &mut ScanScratch| { + scan_one( + &automaton, + &[Vec::new()], + &[], + &exact_specs, + &aliases, + &[], + &contexts, + &[], + text, + 1, + row_index, + scratch, + ) + .expect("SPY remains an observed alias") + }; + + let accepted = scan("SPY — ETF shares rose", 0, &mut scratch); + assert_eq!(accepted.1, ARTICLE_ACCEPTED); + assert_eq!(accepted.2, EXP_SUPPORTED); + + let rejected = scan("SPY ETF appeared beside a spy plane", 1, &mut scratch); + assert_eq!(rejected.1, ARTICLE_REJECTED); + assert_eq!(rejected.2, EXP_NOT_SUPPORTED); + + let missing = scan("SPY reported quarterly earnings", 2, &mut scratch); + assert_eq!(missing.1, ARTICLE_REJECTED); + assert_eq!(missing.2, EXP_NOT_SUPPORTED); + } + + #[test] + fn representative_fed_gdp_and_sec_contexts_keep_frozen_behavior() { + let automaton = + build_automaton(&["federal reserve".to_string()]).expect("valid normalized automaton"); + let pattern_consumers = vec![vec![0]]; + let consumers = vec![ConsumerSpec { + kind: LEXICAL, + a: 0, + b: 0, + c: 0, + }]; + let exact_specs = ExactSpecs::new(vec![ + ExactSpec { + value: "GDP".chars().collect(), + mode: ExactMode::CaseSensitive, + kind: LEXICAL, + index: 1, + }, + ExactSpec { + value: "SEC".chars().collect(), + mode: ExactMode::CaseSensitive, + kind: LEXICAL, + index: 2, + }, + ]) + .expect("valid exact lexical matchers"); + let lexical = vec![ + LexicalSpec { + context_window_chars: 0, + context_terms: Vec::new(), + is_event: false, + market_wide_authorized: true, + unresolved_tag: false, + has_event_family: false, + }, + LexicalSpec { + context_window_chars: 64, + context_terms: vec![b"growth".to_vec(), b"economy".to_vec()], + is_event: false, + market_wide_authorized: true, + unresolved_tag: false, + has_event_family: false, + }, + LexicalSpec { + context_window_chars: 96, + context_terms: vec![b"commission".to_vec(), b"filing".to_vec()], + is_event: false, + market_wide_authorized: false, + unresolved_tag: false, + has_event_family: false, + }, + ]; + let mut scratch = ScanScratch::default(); + let scan = |text: &str, row_index: usize, scratch: &mut ScanScratch| { + scan_one( + &automaton, + &pattern_consumers, + &consumers, + &exact_specs, + &[], + &lexical, + &[], + &[], + text, + 1, + row_index, + scratch, + ) + }; + + let fed = scan("The Federal Reserve cut rates", 0, &mut scratch) + .expect("Federal Reserve is relevant"); + assert_eq!(fed.1, ARTICLE_ACCEPTED); + assert_eq!(fed.5, EXP_SUPPORTED); + + let gdp = scan("Economic growth lifted GDP", 1, &mut scratch) + .expect("contextual GDP is observed"); + assert_eq!(gdp.1, ARTICLE_ACCEPTED); + assert_eq!(gdp.5, EXP_SUPPORTED); + + let gdp_without_context = scan("The GDP widget was renamed", 2, &mut scratch) + .expect("GDP remains an observation when context fails"); + assert_eq!(gdp_without_context.1, ARTICLE_REJECTED); + assert_eq!(gdp_without_context.5, EXP_NOT_SUPPORTED); + + let sec = scan("The SEC filing was published", 3, &mut scratch) + .expect("contextual SEC is observed"); + assert_eq!(sec.1, ARTICLE_ACCEPTED); + assert_eq!(sec.5, EXP_NOT_SUPPORTED); + assert!(scan("SECond place", 4, &mut scratch).is_none()); + } + + #[test] + fn reused_scan_scratch_does_not_leak_candidates_between_rows() { + let automaton = build_automaton(&["alpha".to_string()]).expect("valid automaton"); + let pattern_consumers = vec![vec![0]]; + let consumers = vec![ConsumerSpec { + kind: ALIAS, + a: 0, + b: 0, + c: 0, + }]; + let exact_specs = ExactSpecs::new(Vec::new()).expect("empty exact specs are valid"); + let aliases = vec![AliasSpec { + canonical_group: 0, + context_rule: None, + valid_from_epoch: 0, + valid_to_epoch: None, + family: FAMILY_SP500, + entity_kind: 0, + unresolved_tag: false, + }]; + let mut scratch = ScanScratch::default(); + let scan = |text: &str, row_index: usize, scratch: &mut ScanScratch| { + scan_one( + &automaton, + &pattern_consumers, + &consumers, + &exact_specs, + &aliases, + &[], + &[], + &[], + text, + 1, + row_index, + scratch, + ) + }; + + let first = scan("Alpha rises", 0, &mut scratch).expect("first row matches"); + assert_eq!(first.0, 0); + assert_eq!(first.6, 1); + assert!(scan("nothing relevant", 1, &mut scratch).is_none()); + let third = scan("alpha again", 2, &mut scratch).expect("third row matches"); + assert_eq!(third.0, 2); + assert_eq!(third.6, 1); + } + + #[test] + fn conjunction_marks_resize_reset_and_survive_generation_wrap() { + let mut marks = ConjunctionMarks::default(); + marks.begin_row(130); + for field in 0..4 { + marks.mark(129, field); + } + assert_eq!(marks.touched, [129]); + assert!(marks.is_complete(129)); + + marks.begin_row(130); + assert!(marks.is_empty()); + assert!(!marks.is_complete(129)); + + marks.generation = u32::MAX; + marks.begin_row(130); + assert_eq!(marks.generation, 1); + assert!(marks.stamps.iter().all(|stamp| *stamp == 0)); + assert!(!marks.is_complete(129)); + } + + #[test] + fn sorted_alias_candidates_preserve_longest_ties_and_frozen_counting() { + let automaton = + build_automaton(&["alpha".to_string(), "alpha beta".to_string()]) + .expect("valid automaton"); + let pattern_consumers = vec![vec![0], vec![1, 1, 2]]; + let consumers = vec![ + ConsumerSpec { + kind: ALIAS, + a: 0, + b: 0, + c: 0, + }, + ConsumerSpec { + kind: ALIAS, + a: 1, + b: 0, + c: 0, + }, + ConsumerSpec { + kind: ALIAS, + a: 2, + b: 0, + c: 0, + }, + ]; + let aliases = vec![ + AliasSpec { + canonical_group: 0, + context_rule: None, + valid_from_epoch: 0, + valid_to_epoch: None, + family: FAMILY_SP500, + entity_kind: 0, + unresolved_tag: false, + }, + AliasSpec { + canonical_group: 0, + context_rule: None, + valid_from_epoch: 0, + valid_to_epoch: None, + family: FAMILY_SP500, + entity_kind: 0, + unresolved_tag: false, + }, + // Retained observations are counted before temporal rejection. + AliasSpec { + canonical_group: 0, + context_rule: None, + valid_from_epoch: 10, + valid_to_epoch: None, + family: FAMILY_SP500, + entity_kind: 0, + unresolved_tag: false, + }, + ]; + let exact_specs = ExactSpecs::new(Vec::new()).expect("empty exact specs are valid"); + let mut scratch = ScanScratch::default(); + let observed = scan_one( + &automaton, + &pattern_consumers, + &consumers, + &exact_specs, + &aliases, + &[], + &[], + &[], + "alpha beta", + 1, + 0, + &mut scratch, + ) + .expect("long aliases match"); + + assert_eq!(observed.1, ARTICLE_ACCEPTED); + assert_eq!(observed.2, EXP_SUPPORTED); + assert_eq!(observed.6, 2); + } + + #[test] + fn sorted_lexical_candidates_deduplicate_per_rule_and_span() { + let automaton = build_automaton(&["signal".to_string()]).expect("valid automaton"); + let pattern_consumers = vec![vec![0, 0]]; + let consumers = vec![ConsumerSpec { + kind: LEXICAL, + a: 0, + b: 0, + c: 0, + }]; + let lexical = vec![LexicalSpec { + context_window_chars: 0, + context_terms: Vec::new(), + is_event: true, + market_wide_authorized: false, + unresolved_tag: false, + has_event_family: true, + }]; + let exact_specs = ExactSpecs::new(Vec::new()).expect("empty exact specs are valid"); + let mut scratch = ScanScratch::default(); + let observed = scan_one( + &automaton, + &pattern_consumers, + &consumers, + &exact_specs, + &[], + &lexical, + &[], + &[], + "signal and signal", + 1, + 0, + &mut scratch, + ) + .expect("lexical events match"); + + assert_eq!(observed.1, ARTICLE_ACCEPTED); + assert_eq!(observed.6, 2); + } + + #[test] + fn conjunction_masks_count_once_and_do_not_leak_between_rows() { + let automaton = build_automaton(&[ + "commission".to_string(), + "adopted".to_string(), + "rule".to_string(), + "market".to_string(), + ]) + .expect("valid automaton"); + let pattern_consumers = vec![vec![0, 0], vec![1, 1], vec![2], vec![3]]; + let consumers = vec![ + ConsumerSpec { + kind: CONJUNCTION, + a: 0, + b: 0, + c: 0, + }, + ConsumerSpec { + kind: CONJUNCTION, + a: 0, + b: 1, + c: 0, + }, + ConsumerSpec { + kind: CONJUNCTION, + a: 0, + b: 2, + c: 0, + }, + ConsumerSpec { + kind: CONJUNCTION, + a: 0, + b: 3, + c: 0, + }, + ]; + let exact_specs = ExactSpecs::new(vec![ExactSpec { + value: "SEC".chars().collect(), + mode: ExactMode::CaseSensitive, + kind: CONJUNCTION, + index: 0, + }]) + .expect("valid exact actor"); + let conjunctions = vec![ConjunctionSpec { + actor_phrase_count: 1, + action_phrase_count: 1, + object_phrase_count: 1, + scope_phrase_count: 1, + }]; + let mut scratch = ScanScratch::default(); + let scan = |text: &str, row_index: usize, scratch: &mut ScanScratch| { + scan_one( + &automaton, + &pattern_consumers, + &consumers, + &exact_specs, + &[], + &[], + &[], + &conjunctions, + text, + 1, + row_index, + scratch, + ) + }; + + let first = scan( + "SEC commission adopted adopted rule market", + 0, + &mut scratch, + ) + .expect("complete conjunction"); + assert_eq!(first.6, 1); + assert_eq!(first.5, EXP_SUPPORTED); + + assert!(scan("SEC adopted rule", 1, &mut scratch).is_none()); + let third = scan("commission adopted rule market", 2, &mut scratch) + .expect("later complete conjunction"); + assert_eq!(third.6, 1); + } +}