use aho_corasick::{AhoCorasick, AhoCorasickBuilder, AhoCorasickKind, MatchKind};
use pyo3::exceptions::PyValueError;
use pyo3::prelude::*;
use rayon::prelude::*;
use rayon::ThreadPoolBuilder;
use std::collections::HashMap;

static UNICODE_INDEX: &[u8] = include_bytes!("generated/unicode_index.bin");
static UNICODE_POOL: &[u8] = include_bytes!("generated/unicode_pool.bin");
static PYTHON_WHITESPACE: &[u8] = include_bytes!("generated/python_whitespace.bin");

const ALIAS: u8 = 0;
const LEXICAL: u8 = 1;
const CONJUNCTION: u8 = 2;

const FAMILY_NONE: i8 = -1;
const FAMILY_SP500: i8 = 0;
const FAMILY_NASDAQ100: i8 = 1;
const FAMILY_RUSSELL2000: i8 = 2;
const FAMILY_MARKET_WIDE: i8 = 3;

const ARTICLE_ACCEPTED: u8 = 0;
const ARTICLE_REJECTED: u8 = 2;
const EXP_SUPPORTED: u8 = 0;
const EXP_PLAUSIBLE: u8 = 1;
const EXP_NOT_SUPPORTED: u8 = 2;

const NORMALIZED_DFA_MAX_PATTERNS: usize = 512;
const NORMALIZED_DFA_MAX_PATTERN_BYTES: usize = 16 * 1024;
const NORMALIZED_DFA_MAX_HEAP_BYTES: usize = 32 * 1024 * 1024;

fn try_standard_automaton(
    patterns: &[String],
    ascii_case_insensitive: bool,
    kind: Option<AhoCorasickKind>,
) -> Result<AhoCorasick, String> {
    if patterns.iter().any(|pattern| pattern.is_empty()) {
        return Err("patterns must be non-empty".to_string());
    }
    let mut builder = AhoCorasickBuilder::new();
    builder
        .match_kind(MatchKind::Standard)
        .ascii_case_insensitive(ascii_case_insensitive)
        .prefilter(true);
    if let Some(kind) = kind {
        builder.kind(Some(kind));
    }
    builder.build(patterns).map_err(|error| error.to_string())
}

fn build_auto_automaton(
    patterns: &[String],
    ascii_case_insensitive: bool,
) -> PyResult<AhoCorasick> {
    try_standard_automaton(patterns, ascii_case_insensitive, None)
        .map_err(PyValueError::new_err)
}

fn normalized_dfa_eligible(patterns: &[String]) -> bool {
    patterns.len() <= NORMALIZED_DFA_MAX_PATTERNS
        && patterns
            .iter()
            .try_fold(0usize, |total, pattern| total.checked_add(pattern.len()))
            .is_some_and(|total| total <= NORMALIZED_DFA_MAX_PATTERN_BYTES)
}

fn build_automaton(patterns: &[String]) -> PyResult<AhoCorasick> {
    if normalized_dfa_eligible(patterns) {
        if let Ok(dfa) =
            try_standard_automaton(patterns, false, Some(AhoCorasickKind::DFA))
        {
            if dfa.memory_usage() <= NORMALIZED_DFA_MAX_HEAP_BYTES {
                return Ok(dfa);
            }
        }
    }
    build_auto_automaton(patterns, false)
}

#[derive(Clone)]
struct ConsumerSpec {
    kind: u8,
    a: usize,
    b: usize,
    c: usize,
}

#[derive(Clone, Copy, Eq, PartialEq)]
enum ExactMode {
    CaseSensitive,
    PythonIgnoreCaseAsciiLiteral,
}

#[derive(Clone)]
struct ExactSpec {
    value: Vec<char>,
    mode: ExactMode,
    kind: u8,
    index: usize,
}

struct ExactAutomaton {
    automaton: AhoCorasick,
    pattern_specs: Vec<Vec<usize>>,
}

struct ExactSpecs {
    specs: Vec<ExactSpec>,
    case_sensitive: Option<ExactAutomaton>,
    ignorecase_ascii: Option<ExactAutomaton>,
}

#[derive(Clone, Copy)]
struct ExactByteMatch {
    spec_index: usize,
    start: usize,
    end: usize,
}

#[derive(Clone)]
struct ContextRule {
    window_chars: usize,
    accept_clauses: Vec<Vec<Vec<u8>>>,
    reject_terms: Vec<Vec<u8>>,
}

#[derive(Clone)]
struct AliasSpec {
    canonical_group: usize,
    context_rule: Option<usize>,
    valid_from_epoch: i64,
    valid_to_epoch: Option<i64>,
    family: i8,
    entity_kind: u8,
    unresolved_tag: bool,
}

#[derive(Clone)]
struct LexicalSpec {
    context_window_chars: usize,
    context_terms: Vec<Vec<u8>>,
    is_event: bool,
    market_wide_authorized: bool,
    unresolved_tag: bool,
    has_event_family: bool,
}

#[derive(Clone)]
struct ConjunctionSpec {
    actor_phrase_count: usize,
    action_phrase_count: usize,
    object_phrase_count: usize,
    scope_phrase_count: usize,
}

impl ConjunctionSpec {
    #[inline(always)]
    fn contains_phrase(&self, field: usize, phrase_index: usize) -> bool {
        let count = match field {
            0 => self.actor_phrase_count,
            1 => self.action_phrase_count,
            2 => self.object_phrase_count,
            3 => self.scope_phrase_count,
            _ => return false,
        };
        phrase_index < count
    }
}

#[derive(Default)]
struct ConjunctionMarks {
    generation: u32,
    stamps: Vec<u32>,
    masks: Vec<u8>,
    touched: Vec<usize>,
}

impl ConjunctionMarks {
    fn begin_row(&mut self, rule_count: usize) {
        if self.stamps.len() < rule_count {
            self.stamps.resize(rule_count, 0);
            self.masks.resize(rule_count, 0);
        }
        self.touched.clear();
        self.generation = self.generation.wrapping_add(1);
        if self.generation == 0 {
            self.stamps.fill(0);
            self.generation = 1;
        }
    }

    #[inline(always)]
    fn mark(&mut self, rule_index: usize, field: usize) {
        if rule_index >= self.stamps.len() || field >= 4 {
            return;
        }
        if self.stamps[rule_index] != self.generation {
            self.stamps[rule_index] = self.generation;
            self.masks[rule_index] = 0;
            self.touched.push(rule_index);
        }
        self.masks[rule_index] |= 1u8 << field;
    }

    #[inline(always)]
    fn is_complete(&self, rule_index: usize) -> bool {
        self.stamps[rule_index] == self.generation && self.masks[rule_index] == 0b1111
    }

    #[inline(always)]
    fn is_empty(&self) -> bool {
        self.touched.is_empty()
    }
}

// (row_index, article_status, sp500, nasdaq100, russell2000, market_wide,
//  observation_count, native_candidate, exact_candidate)
type FactRow = (usize, u8, u8, u8, u8, u8, usize, bool, bool);

#[derive(Default)]
struct ScanScratch {
    chars: Vec<char>,
    normalized: Vec<u8>,
    offsets: Vec<usize>,
    exact_byte_matches: Vec<ExactByteMatch>,
    exact_matches: Vec<(usize, usize, usize)>,
    exact_boundaries: Vec<usize>,
    alias_candidates: Vec<(usize, usize, usize)>,
    lexical_candidates: Vec<(usize, usize, usize)>,
    conjunction_marks: ConjunctionMarks,
}

impl ScanScratch {
    fn reset(&mut self, conjunction_count: usize) {
        self.chars.clear();
        self.normalized.clear();
        self.offsets.clear();
        self.exact_byte_matches.clear();
        self.exact_matches.clear();
        self.exact_boundaries.clear();
        self.alias_candidates.clear();
        self.lexical_candidates.clear();
        self.conjunction_marks.begin_row(conjunction_count);
    }
}

#[inline(always)]
fn is_python_ascii_space(byte: u8) -> bool {
    matches!(
        byte,
        b' ' | b'\t' | b'\n' | 0x0b | 0x0c | b'\r' | 0x1c..=0x1f
    )
}

#[inline(always)]
fn is_separator_ascii(byte: u8) -> bool {
    matches!(
        byte,
        b'-' | b'.' | b',' | b':' | b';' | b'(' | b')' | b'[' | b']' | b'{' | b'}' | b'/'
    )
}

#[inline(always)]
fn is_token_byte(byte: u8) -> bool {
    byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'_'
}

#[inline(always)]
fn is_exact_token_char(ch: char, mode: ExactMode) -> bool {
    if ch.is_ascii_alphanumeric() || ch == '_' {
        return true;
    }
    // Python's Unicode-aware re.IGNORECASE expands [A-Za-z] to these four
    // additional code points. They are token characters only for the
    // python_ignorecase_ascii_literal mode; applying this globally would change
    // the frozen boundary behavior of case-sensitive exact aliases.
    matches!(mode, ExactMode::PythonIgnoreCaseAsciiLiteral)
        && matches!(ch, '\u{0130}' | '\u{0131}' | '\u{017f}' | '\u{212a}')
}

#[inline(always)]
fn is_python_space(ch: char) -> bool {
    let cp = ch as usize;
    let byte = PYTHON_WHITESPACE[cp >> 3];
    (byte & (1 << (cp & 7))) != 0
}

#[inline(always)]
fn mapped_bytes(ch: char) -> Option<&'static [u8]> {
    let cp = ch as usize;
    let base = cp * 4;
    let offset = u32::from_le_bytes([
        UNICODE_INDEX[base],
        UNICODE_INDEX[base + 1],
        UNICODE_INDEX[base + 2],
        UNICODE_INDEX[base + 3],
    ]) as usize;
    if offset == 0 {
        return None;
    }
    let len = u16::from_le_bytes([UNICODE_POOL[offset], UNICODE_POOL[offset + 1]]) as usize;
    Some(&UNICODE_POOL[offset + 2..offset + 2 + len])
}

#[inline(always)]
fn normalize_post_char(
    ch: char,
    source_index: usize,
    normalized: &mut Vec<u8>,
    offsets: &mut Vec<usize>,
    pending_space: &mut bool,
) {
    let ch = match ch {
        '-' | '‐' | '‑' | '‒' | '–' | '—' | '―' | '−' => ' ',
        '.' | ',' | ':' | ';' | '(' | ')' | '[' | ']' | '{' | '}' | '／' | '/' => ' ',
        '\'' | '’' | '‘' | '‛' | '`' | '´' => '\'',
        other => other,
    };
    if is_python_space(ch) {
        *pending_space = !normalized.is_empty();
        return;
    }
    if *pending_space {
        normalized.push(b' ');
        offsets.push(source_index);
        *pending_space = false;
    }
    let mut buffer = [0u8; 4];
    let encoded = ch.encode_utf8(&mut buffer).as_bytes();
    normalized.extend_from_slice(encoded);
    offsets.extend(std::iter::repeat(source_index).take(encoded.len()));
}

fn normalize_source_char(
    source_character: char,
    source_index: usize,
    normalized: &mut Vec<u8>,
    offsets: &mut Vec<usize>,
    pending_space: &mut bool,
) {
    if source_character.is_ascii() {
        let mut byte = (source_character as u8).to_ascii_lowercase();
        if is_separator_ascii(byte) {
            byte = b' ';
        } else if byte == b'`' {
            byte = b'\'';
        }
        if is_python_ascii_space(byte) {
            *pending_space = !normalized.is_empty();
            return;
        }
        if *pending_space {
            normalized.push(b' ');
            offsets.push(source_index);
            *pending_space = false;
        }
        normalized.push(byte);
        offsets.push(source_index);
    } else if let Some(mapped) = mapped_bytes(source_character) {
        let mapped = std::str::from_utf8(mapped).expect("generated Unicode pool is valid UTF-8");
        for character in mapped.chars() {
            normalize_post_char(character, source_index, normalized, offsets, pending_space);
        }
    } else {
        normalize_post_char(
            source_character,
            source_index,
            normalized,
            offsets,
            pending_space,
        );
    }
}

#[cfg(test)]
fn normalize_chars_into(
    chars: &[char],
    source_base: usize,
    normalized: &mut Vec<u8>,
    offsets: &mut Vec<usize>,
) {
    normalized.clear();
    offsets.clear();
    normalized.reserve(chars.len());
    offsets.reserve(chars.len());
    let mut pending_space = false;
    for (local_index, source_character) in chars.iter().copied().enumerate() {
        let source_index = source_base + local_index;
        normalize_source_char(
            source_character,
            source_index,
            normalized,
            offsets,
            &mut pending_space,
        );
    }
}

#[inline(always)]
fn normalize_ascii_byte(mut byte: u8) -> u8 {
    byte.make_ascii_lowercase();
    if is_separator_ascii(byte) {
        b' '
    } else if byte == b'`' {
        b'\''
    } else {
        byte
    }
}

fn normalize_ascii_text_into(text: &str, normalized: &mut Vec<u8>, offsets: &mut Vec<usize>) {
    debug_assert!(text.is_ascii());
    let mut pending_space = false;
    for (source_index, source_byte) in text.bytes().enumerate() {
        let byte = normalize_ascii_byte(source_byte);
        if is_python_ascii_space(byte) {
            pending_space = !normalized.is_empty();
            continue;
        }
        if pending_space {
            normalized.push(b' ');
            offsets.push(source_index);
            pending_space = false;
        }
        normalized.push(byte);
        offsets.push(source_index);
    }
}

// Phase31A mixed-row block candidate; whole-ASCII control is unchanged.
// Derived from a2dbcf9's Phase26 transform and scalar collapse-mask emitter.
// The caller retains the frozen Unicode normalizer and whole-ASCII control.

#[cfg(target_arch = "x86_64")]
fn mixed_ascii_avx2_available() -> bool {
    static AVAILABLE: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
    *AVAILABLE.get_or_init(|| std::arch::is_x86_feature_detected!("avx2"))
}

#[cfg(target_arch = "x86_64")]
const MIXED_ASCII_COLLAPSE_BY_LOW_NIBBLE: [u8; 16] =
    [4, 0, 0, 0, 0, 0, 0, 0, 4, 5, 9, 169, 7, 167, 6, 6];

#[cfg(target_arch = "x86_64")]
const MIXED_ASCII_HIGH_NIBBLE_BITS: [u8; 16] =
    [1, 2, 4, 8, 16, 32, 64, 128, 0, 0, 0, 0, 0, 0, 0, 0];

#[cfg(target_arch = "x86_64")]
#[inline(always)]
fn emit_mixed_ascii_block(
    mapped: &[u8],
    collapse_mask: u32,
    source_base: usize,
    normalized: &mut Vec<u8>,
    offsets: &mut Vec<usize>,
    pending_space: &mut bool,
) {
    debug_assert!(mapped.len() <= 32);
    let mut local_index = 0usize;
    while local_index < mapped.len() {
        let remaining_mask = collapse_mask >> local_index;
        let remaining_len = mapped.len() - local_index;
        if remaining_mask & 1 != 0 {
            let run_len = (remaining_mask.trailing_ones() as usize).min(remaining_len);
            *pending_space = !normalized.is_empty();
            local_index += run_len;
            continue;
        }
        let run_len = (remaining_mask.trailing_zeros() as usize).min(remaining_len);
        if *pending_space {
            normalized.push(b' ');
            offsets.push(source_base + local_index);
            *pending_space = false;
        }
        let run_end = local_index + run_len;
        normalized.extend_from_slice(&mapped[local_index..run_end]);
        offsets.extend(source_base + local_index..source_base + run_end);
        local_index = run_end;
    }
}

#[cfg(target_arch = "x86_64")]
#[target_feature(enable = "avx2")]
unsafe fn normalize_mixed_ascii_prefix_avx2_into(
    bytes: &[u8],
    source_base: usize,
    normalized: &mut Vec<u8>,
    offsets: &mut Vec<usize>,
    pending_space: &mut bool,
) -> usize {
    use std::arch::x86_64::*;

    // Every load is within the provided slice. A vector may contain UTF-8,
    // but only the ASCII prefix before its first high byte may be emitted.
    // The caller checks AVX2 support before entering this target-feature body.
    let low_table = _mm256_broadcastsi128_si256(unsafe {
        _mm_loadu_si128(MIXED_ASCII_COLLAPSE_BY_LOW_NIBBLE.as_ptr().cast())
    });
    let high_table = _mm256_broadcastsi128_si256(unsafe {
        _mm_loadu_si128(MIXED_ASCII_HIGH_NIBBLE_BITS.as_ptr().cast())
    });
    let low_nibble_mask = _mm256_set1_epi8(0x0f);
    let lowercase_bit = _mm256_set1_epi8(0x20);
    let upper_lower_bound = _mm256_set1_epi8((b'A' - 1) as i8);
    let upper_upper_bound = _mm256_set1_epi8((b'Z' + 1) as i8);
    let backtick = _mm256_set1_epi8(b'`' as i8);
    let apostrophe = _mm256_set1_epi8(b'\'' as i8);
    let mut mapped = [0u8; 32];
    let mut consumed = 0usize;

    while bytes.len() - consumed >= 32 {
        let source = unsafe { _mm256_loadu_si256(bytes.as_ptr().add(consumed).cast()) };
        let high_mask = _mm256_movemask_epi8(source) as u32;
        let ascii_prefix = high_mask.trailing_zeros() as usize;
        if ascii_prefix == 0 {
            break;
        }
        let low_nibbles = _mm256_and_si256(source, low_nibble_mask);
        let high_nibbles = _mm256_and_si256(_mm256_srli_epi16(source, 4), low_nibble_mask);
        let allowed_high_bits = _mm256_shuffle_epi8(low_table, low_nibbles);
        let observed_high_bits = _mm256_shuffle_epi8(high_table, high_nibbles);
        let collapse = _mm256_cmpeq_epi8(
            _mm256_and_si256(allowed_high_bits, observed_high_bits),
            observed_high_bits,
        );
        let uppercase = _mm256_and_si256(
            _mm256_cmpgt_epi8(source, upper_lower_bound),
            _mm256_cmpgt_epi8(upper_upper_bound, source),
        );
        let lowercased = _mm256_or_si256(source, _mm256_and_si256(uppercase, lowercase_bit));
        let normalized_backtick = _mm256_blendv_epi8(
            lowercased,
            apostrophe,
            _mm256_cmpeq_epi8(source, backtick),
        );
        unsafe { _mm256_storeu_si256(mapped.as_mut_ptr().cast(), normalized_backtick) };
        emit_mixed_ascii_block(
            &mapped[..ascii_prefix],
            _mm256_movemask_epi8(collapse) as u32,
            source_base + consumed,
            normalized,
            offsets,
            pending_space,
        );
        consumed += ascii_prefix;
        if ascii_prefix < 32 {
            break;
        }
    }
    consumed
}

fn normalize_text_into(
    text: &str,
    normalized: &mut Vec<u8>,
    offsets: &mut Vec<usize>,
) -> (bool, bool, usize) {
    normalize_text_into_with_mixed_avx2(text, normalized, offsets, true)
}

fn normalize_text_into_with_mixed_avx2(
    text: &str,
    normalized: &mut Vec<u8>,
    offsets: &mut Vec<usize>,
    allow_avx2: bool,
) -> (bool, bool, usize) {
    normalized.clear();
    offsets.clear();
    normalized.reserve(text.len());
    offsets.reserve(text.len());
    if text.is_ascii() {
        normalize_ascii_text_into(text, normalized, offsets);
        return (true, false, text.len());
    }
    // Keep the whole-ASCII path above unchanged. Feature detection is cached
    // and selected once per mixed row, never without the runtime guard.
    #[cfg(target_arch = "x86_64")]
    let use_avx2 = allow_avx2 && mixed_ascii_avx2_available();
    #[cfg(not(target_arch = "x86_64"))]
    let _ = allow_avx2;
    let bytes = text.as_bytes();
    let mut pending_space = false;
    let mut exceptional_ignorecase = false;
    let mut byte_cursor = 0usize;
    let mut source_len = 0usize;
    while byte_cursor < bytes.len() {
        #[cfg(target_arch = "x86_64")]
        if use_avx2 && bytes.len() - byte_cursor >= 32 && bytes[byte_cursor].is_ascii() {
            // SAFETY: use_avx2 includes runtime feature detection. The helper
            // bounds every load and emits only bytes before the first high bit.
            let consumed = unsafe {
                normalize_mixed_ascii_prefix_avx2_into(
                    &bytes[byte_cursor..], source_len, normalized, offsets, &mut pending_space,
                )
            };
            byte_cursor += consumed;
            source_len += consumed; // Every consumed byte is one ASCII scalar.
        }
        // Consume raw ASCII tails once, without decoding their bytes as Unicode
        // scalars or searching the complete run before processing it. Original
        // code-point positions remain separate from UTF-8 byte positions.
        while byte_cursor < bytes.len() && bytes[byte_cursor].is_ascii() {
            let byte = normalize_ascii_byte(bytes[byte_cursor]);
            if is_python_ascii_space(byte) {
                pending_space = !normalized.is_empty();
            } else {
                if pending_space {
                    normalized.push(b' ');
                    offsets.push(source_len);
                    pending_space = false;
                }
                normalized.push(byte);
                offsets.push(source_len);
            }
            byte_cursor += 1;
            source_len += 1;
        }
        if byte_cursor == bytes.len() {
            break;
        }

        // The cursor starts at a character boundary and advances by one byte
        // for ASCII or by the complete decoded scalar's UTF-8 width. Keep the
        // frozen per-scalar mapping and repeated expansion offsets unchanged.
        let source_character = text[byte_cursor..]
            .chars()
            .next()
            .expect("remaining input contains a complete UTF-8 scalar");
        exceptional_ignorecase |= matches!(
            source_character,
            '\u{0130}' | '\u{0131}' | '\u{017f}' | '\u{212a}'
        );
        normalize_source_char(
            source_character,
            source_len,
            normalized,
            offsets,
            &mut pending_space,
        );
        byte_cursor += source_character.len_utf8();
        source_len += 1;
    }
    // The unchanged whole-row ASCII gate already rejected this input.
    (false, exceptional_ignorecase, source_len)
}

#[inline]
fn normalized_contains(haystack: &[u8], needle: &[u8]) -> bool {
    if needle.is_empty() || haystack.len() < needle.len() {
        return false;
    }
    let mut cursor = 0usize;
    while cursor + needle.len() <= haystack.len() {
        if let Some(relative) = haystack[cursor..]
            .windows(needle.len())
            .position(|window| window == needle)
        {
            let start = cursor + relative;
            let end = start + needle.len();
            let left_ok = start == 0 || !is_token_byte(haystack[start - 1]);
            let right_ok = end == haystack.len() || !is_token_byte(haystack[end]);
            if left_ok && right_ok {
                return true;
            }
            cursor = start + 1;
        } else {
            return false;
        }
    }
    false
}

fn normalized_context_window<'a>(
    normalized: &'a [u8],
    offsets: &[usize],
    source_len: usize,
    start: usize,
    end: usize,
    width: usize,
) -> &'a [u8] {
    debug_assert_eq!(normalized.len(), offsets.len());
    debug_assert!(start <= end);
    debug_assert!(end <= source_len);

    let left = start.saturating_sub(width);
    let right = end.saturating_add(width).min(source_len);
    let normalized_left = offsets.partition_point(|&source_index| source_index < left);
    let normalized_right = offsets.partition_point(|&source_index| source_index < right);
    let mut window = &normalized[normalized_left..normalized_right];

    // A full-row normalization can carry a collapsed separator from before the
    // source window to its first non-separator. Normalizing the source window in
    // isolation drops that leading separator, so remove it from the borrowed
    // slice as well. Trailing separators already map to the first source
    // character after the window and are excluded by normalized_right.
    if window.first() == Some(&b' ') {
        window = &window[1..];
    }
    window
}

fn context_status(
    normalized: &[u8],
    offsets: &[usize],
    source_len: usize,
    start: usize,
    end: usize,
    rule: &ContextRule,
) -> u8 {
    // 0=satisfied, 1=explicit_collision_exclusion, 2=context_clause_not_satisfied
    let window = normalized_context_window(
        normalized,
        offsets,
        source_len,
        start,
        end,
        rule.window_chars,
    );
    if rule
        .reject_terms
        .iter()
        .any(|term| normalized_contains(window, term))
    {
        return 1;
    }
    if rule
        .accept_clauses
        .iter()
        .any(|clause| clause.iter().all(|term| normalized_contains(window, term)))
    {
        0
    } else {
        2
    }
}

#[inline(always)]
fn py_ignorecase_ascii_literal_eq(pattern: char, observed: char) -> bool {
    let lower = pattern.to_ascii_lowercase();
    if observed.is_ascii() {
        return observed.to_ascii_lowercase() == lower;
    }
    match lower {
        'i' => matches!(observed, 'İ' | 'ı'),
        's' => observed == 'ſ',
        'k' => observed == 'K',
        _ => false,
    }
}

fn exact_matches(chars: &[char], spec: &ExactSpec) -> Vec<(usize, usize)> {
    let width = spec.value.len();
    if width == 0 || width > chars.len() {
        return Vec::new();
    }
    let mut output = Vec::new();
    for start in 0..=chars.len() - width {
        let end = start + width;
        if start > 0 && is_exact_token_char(chars[start - 1], spec.mode) {
            continue;
        }
        if end < chars.len() && is_exact_token_char(chars[end], spec.mode) {
            continue;
        }
        let matched = spec.value.iter().enumerate().all(|(offset, expected)| {
            let observed = chars[start + offset];
            match spec.mode {
                ExactMode::CaseSensitive => observed == *expected,
                ExactMode::PythonIgnoreCaseAsciiLiteral => {
                    py_ignorecase_ascii_literal_eq(*expected, observed)
                }
            }
        });
        if matched {
            output.push((start, end));
        }
    }
    output
}

#[inline(always)]
fn exact_boundaries_match(text: &str, start: usize, end: usize, mode: ExactMode) -> bool {
    let left_ok = start == 0
        || !is_exact_token_char(
            text[..start]
                .chars()
                .next_back()
                .expect("nonempty exact left boundary"),
            mode,
        );
    let right_ok = end == text.len()
        || !is_exact_token_char(
            text[end..]
                .chars()
                .next()
                .expect("nonempty exact right boundary"),
            mode,
        );
    left_ok && right_ok
}

impl ExactAutomaton {
    fn append_matches(&self, text: &str, mode: ExactMode, output: &mut Vec<ExactByteMatch>) {
        for matched in self.automaton.find_overlapping_iter(text.as_bytes()) {
            let start = matched.start();
            let end = matched.end();
            if !exact_boundaries_match(text, start, end, mode) {
                continue;
            }
            for &spec_index in &self.pattern_specs[matched.pattern().as_usize()] {
                output.push(ExactByteMatch {
                    spec_index,
                    start,
                    end,
                });
            }
        }
    }
}

fn build_exact_automaton(
    specs: &[ExactSpec],
    mode: ExactMode,
    ascii_case_insensitive: bool,
) -> PyResult<Option<ExactAutomaton>> {
    let mut patterns = Vec::<String>::new();
    let mut pattern_specs = Vec::<Vec<usize>>::new();
    let mut pattern_ids = HashMap::<String, usize>::new();

    for (spec_index, spec) in specs.iter().enumerate() {
        if spec.mode != mode {
            continue;
        }
        let value: String = spec.value.iter().collect();
        let pattern_key = if ascii_case_insensitive {
            value.to_ascii_lowercase()
        } else {
            value.clone()
        };
        if let Some(&pattern_id) = pattern_ids.get(&pattern_key) {
            pattern_specs[pattern_id].push(spec_index);
        } else {
            let pattern_id = patterns.len();
            pattern_ids.insert(pattern_key, pattern_id);
            patterns.push(value);
            pattern_specs.push(vec![spec_index]);
        }
    }

    if patterns.is_empty() {
        return Ok(None);
    }
    let automaton = build_auto_automaton(&patterns, ascii_case_insensitive)?;
    Ok(Some(ExactAutomaton {
        automaton,
        pattern_specs,
    }))
}

impl ExactSpecs {
    fn new(specs: Vec<ExactSpec>) -> PyResult<Self> {
        let case_sensitive = build_exact_automaton(&specs, ExactMode::CaseSensitive, false)?;
        let ignorecase_ascii =
            build_exact_automaton(&specs, ExactMode::PythonIgnoreCaseAsciiLiteral, true)?;
        Ok(Self {
            specs,
            case_sensitive,
            ignorecase_ascii,
        })
    }

    fn append_matching(
        &self,
        text: &str,
        byte_matches: &mut Vec<ExactByteMatch>,
        output: &mut Vec<(usize, usize, usize)>,
        exceptional_chars: &mut Vec<char>,
        boundaries: &mut Vec<usize>,
        text_is_ascii: bool,
        exceptional_ignorecase: bool,
    ) {
        byte_matches.clear();
        output.clear();
        exceptional_chars.clear();
        boundaries.clear();
        if let Some(automaton) = &self.case_sensitive {
            automaton.append_matches(text, ExactMode::CaseSensitive, byte_matches);
        }

        if !exceptional_ignorecase {
            if let Some(automaton) = &self.ignorecase_ascii {
                automaton.append_matches(
                    text,
                    ExactMode::PythonIgnoreCaseAsciiLiteral,
                    byte_matches,
                );
            }
        }

        output.reserve(byte_matches.len());
        if text_is_ascii {
            output.extend(
                byte_matches
                    .iter()
                    .copied()
                    .map(|matched| (matched.spec_index, matched.start, matched.end)),
            );
        } else if !byte_matches.is_empty() {
            // Aho-Corasick reports UTF-8 byte offsets while the frozen Python
            // surface reports code-point offsets. Build this map only for the
            // uncommon non-ASCII row that actually produced an exact hit.
            boundaries.extend(text.char_indices().map(|(offset, _)| offset));
            boundaries.push(text.len());
            output.extend(byte_matches.iter().copied().map(|matched| {
                let start = boundaries
                    .binary_search(&matched.start)
                    .expect("exact match starts at a UTF-8 character boundary");
                let end = boundaries
                    .binary_search(&matched.end)
                    .expect("exact match ends at a UTF-8 character boundary");
                (matched.spec_index, start, end)
            }));
        }

        if exceptional_ignorecase {
            // Python re.IGNORECASE gives ASCII I/S/K four additional Unicode
            // equivalents (İ, ı, ſ, K). The common ASCII automaton cannot change
            // byte width while matching, so retain the exact scalar fallback
            // only for rows that contain one of those exceptional code points.
            exceptional_chars.extend(text.chars());
            for (spec_index, spec) in self.specs.iter().enumerate() {
                if spec.mode != ExactMode::PythonIgnoreCaseAsciiLiteral {
                    continue;
                }
                output.extend(
                    exact_matches(exceptional_chars, spec)
                        .into_iter()
                        .map(|(start, end)| (spec_index, start, end)),
                );
            }
        }
    }

    #[cfg(test)]
    fn matching(&self, text: &str) -> Vec<(usize, usize, usize)> {
        let mut byte_matches = Vec::new();
        let mut output = Vec::new();
        let mut exceptional_chars = Vec::new();
        let mut boundaries = Vec::new();
        let mut normalized = Vec::new();
        let mut offsets = Vec::new();
        let (text_is_ascii, exceptional_ignorecase, _) =
            normalize_text_into(text, &mut normalized, &mut offsets);
        self.append_matching(
            text,
            &mut byte_matches,
            &mut output,
            &mut exceptional_chars,
            &mut boundaries,
            text_is_ascii,
            exceptional_ignorecase,
        );
        output
    }
}

fn scan_one(
    automaton: &AhoCorasick,
    pattern_consumers: &[Vec<usize>],
    consumers: &[ConsumerSpec],
    exact_specs: &ExactSpecs,
    aliases: &[AliasSpec],
    lexical: &[LexicalSpec],
    contexts: &[ContextRule],
    conjunctions: &[ConjunctionSpec],
    text: &str,
    observed_epoch: i64,
    row_index: usize,
    scratch: &mut ScanScratch,
) -> Option<FactRow> {
    scratch.reset(conjunctions.len());
    let (text_is_ascii, exceptional_ignorecase, source_len) =
        normalize_text_into(text, &mut scratch.normalized, &mut scratch.offsets);
    let mut native_candidate = false;
    if !scratch.normalized.is_empty() {
        for matched in automaton.find_overlapping_iter(&scratch.normalized) {
            let start = matched.start();
            let end = matched.end();
            let left_ok = start == 0 || !is_token_byte(scratch.normalized[start - 1]);
            let right_ok =
                end == scratch.normalized.len() || !is_token_byte(scratch.normalized[end]);
            if !left_ok || !right_ok {
                continue;
            }
            native_candidate = true;
            let original_start = scratch.offsets[start];
            let original_end = scratch.offsets[end - 1] + 1;
            for &consumer_id in &pattern_consumers[matched.pattern().as_usize()] {
                let spec = &consumers[consumer_id];
                match spec.kind {
                    ALIAS => {
                        scratch
                            .alias_candidates
                            .push((spec.a, original_start, original_end));
                    }
                    LEXICAL => {
                        scratch
                            .lexical_candidates
                            .push((spec.a, original_start, original_end));
                    }
                    CONJUNCTION => {
                        if conjunctions
                            .get(spec.a)
                            .is_some_and(|rule| rule.contains_phrase(spec.b, spec.c))
                        {
                            scratch.conjunction_marks.mark(spec.a, spec.b);
                        }
                    }
                    _ => unreachable!("validated consumer kind"),
                }
            }
        }
    }

    let mut exact_candidate = false;
    exact_specs.append_matching(
        text,
        &mut scratch.exact_byte_matches,
        &mut scratch.exact_matches,
        &mut scratch.chars,
        &mut scratch.exact_boundaries,
        text_is_ascii,
        exceptional_ignorecase,
    );
    for &(spec_index, start, end) in &scratch.exact_matches {
        exact_candidate = true;
        let spec = &exact_specs.specs[spec_index];
        match spec.kind {
            ALIAS => scratch.alias_candidates.push((spec.index, start, end)),
            LEXICAL => scratch.lexical_candidates.push((spec.index, start, end)),
            CONJUNCTION => {
                scratch.conjunction_marks.mark(spec.index, 0);
            }
            _ => unreachable!("validated exact kind"),
        }
    }

    if scratch.alias_candidates.is_empty()
        && scratch.lexical_candidates.is_empty()
        && scratch.conjunction_marks.is_empty()
    {
        return None;
    }

    let mut accepted_nonsemantic = false;
    let mut accepted_evidence = false;
    let mut unresolved_exposure = false;
    let mut unresolved_event = false;
    let mut supported_mask: u8 = 0;
    let mut observation_count = 0usize;

    // Group aliases by (canonical_id, start) without a per-row hash table. The
    // longest end in each group wins, while distinct aliases that tie remain
    // distinct observations. Exact duplicate candidates collapse here.
    scratch.alias_candidates.sort_unstable_by_key(
        |&(alias_index, start, end)| {
            (aliases[alias_index].canonical_group, start, end, alias_index)
        },
    );
    scratch.alias_candidates.dedup();
    let mut group_start = 0usize;
    while group_start < scratch.alias_candidates.len() {
        let (first_alias, first_start, _) = scratch.alias_candidates[group_start];
        let canonical_group = aliases[first_alias].canonical_group;
        let mut group_end = group_start + 1;
        let mut longest_end = scratch.alias_candidates[group_start].2;
        while group_end < scratch.alias_candidates.len() {
            let (alias_index, start, end) = scratch.alias_candidates[group_end];
            if aliases[alias_index].canonical_group != canonical_group || start != first_start {
                break;
            }
            longest_end = longest_end.max(end);
            group_end += 1;
        }

        for candidate_index in group_start..group_end {
            let (alias_index, start, end) = scratch.alias_candidates[candidate_index];
            if end < longest_end {
                continue;
            }
            let spec = &aliases[alias_index];

            // The frozen compact contract counts a retained observation even
            // when its temporal or context checks later reject it.
            observation_count += 1;
            let active = observed_epoch >= spec.valid_from_epoch
                && spec
                    .valid_to_epoch
                    .map(|until| observed_epoch < until)
                    .unwrap_or(true);
            if !active {
                continue;
            }

            let context_code = if let Some(rule_index) = spec.context_rule {
                context_status(
                    &scratch.normalized,
                    &scratch.offsets,
                    source_len,
                    start,
                    end,
                    &contexts[rule_index],
                )
            } else {
                0
            };
            if context_code != 0 {
                // Frozen exposure unresolved filter excludes explicit collision and
                // context-clause failures. Missing context cannot occur because the
                // validated config rejects unknown rules during construction.
                continue;
            }

            accepted_nonsemantic = true;
            if spec.family != FAMILY_NONE && (spec.entity_kind == 0 || spec.entity_kind == 1) {
                accepted_evidence = true;
                supported_mask |= 1u8 << (spec.family as u8);
            } else if spec.unresolved_tag {
                unresolved_exposure = true;
            }
        }
        group_start = group_end;
    }

    // Observation identity for lexical matches is (rule, start, end), so the
    // sorted sparse candidates replace the former observation HashSet exactly.
    scratch.lexical_candidates.sort_unstable();
    scratch.lexical_candidates.dedup();
    for candidate_index in 0..scratch.lexical_candidates.len() {
        let (lexical_index, start, end) = scratch.lexical_candidates[candidate_index];
        let spec = &lexical[lexical_index];
        observation_count += 1;

        let context_ok = if spec.context_terms.is_empty() {
            true
        } else {
            let window = normalized_context_window(
                &scratch.normalized,
                &scratch.offsets,
                source_len,
                start,
                end,
                spec.context_window_chars,
            );
            spec.context_terms
                .iter()
                .any(|term| normalized_contains(window, term))
        };

        if spec.is_event {
            if spec.market_wide_authorized {
                accepted_evidence = true;
                supported_mask |= 1u8 << FAMILY_MARKET_WIDE;
            } else {
                if spec.unresolved_tag {
                    unresolved_exposure = true;
                }
                if spec.has_event_family {
                    unresolved_event = true;
                }
            }
        } else {
            if context_ok {
                accepted_nonsemantic = true;
            }
            if spec.market_wide_authorized && context_ok {
                accepted_evidence = true;
                supported_mask |= 1u8 << FAMILY_MARKET_WIDE;
            } else if spec.unresolved_tag {
                // Matches frozen generic lexical unresolved branch, including a
                // lexical context failure, whose evidence reason is the generic
                // identity-resolved-without-path code.
                unresolved_exposure = true;
            }
        }
    }

    for touched_index in 0..scratch.conjunction_marks.touched.len() {
        let rule_index = scratch.conjunction_marks.touched[touched_index];
        if scratch.conjunction_marks.is_complete(rule_index) {
            // Only one conjunction observation exists per configured rule.
            observation_count += 1;
            accepted_evidence = true;
            supported_mask |= 1u8 << FAMILY_MARKET_WIDE;
        }
    }

    if observation_count == 0 {
        return None;
    }
    let article_status = if accepted_evidence || accepted_nonsemantic || unresolved_event {
        ARTICLE_ACCEPTED
    } else {
        ARTICLE_REJECTED
    };
    let completed_elsewhere = supported_mask != 0;
    let exposure_status = |family: i8| -> u8 {
        if (supported_mask & (1u8 << (family as u8))) != 0 {
            EXP_SUPPORTED
        } else if family != FAMILY_MARKET_WIDE && !completed_elsewhere && unresolved_exposure {
            EXP_PLAUSIBLE
        } else {
            EXP_NOT_SUPPORTED
        }
    };

    Some((
        row_index,
        article_status,
        exposure_status(FAMILY_SP500),
        exposure_status(FAMILY_NASDAQ100),
        exposure_status(FAMILY_RUSSELL2000),
        exposure_status(FAMILY_MARKET_WIDE),
        observation_count,
        native_candidate,
        exact_candidate,
    ))
}

#[pyclass(frozen, module = "range_curate._native")]
struct NativeCompactFactScanner {
    automaton: AhoCorasick,
    pattern_consumers: Vec<Vec<usize>>,
    consumers: Vec<ConsumerSpec>,
    exact_specs: ExactSpecs,
    aliases: Vec<AliasSpec>,
    lexical: Vec<LexicalSpec>,
    contexts: Vec<ContextRule>,
    conjunctions: Vec<ConjunctionSpec>,
}

#[pymethods]
impl NativeCompactFactScanner {
    #[new]
    #[pyo3(signature = (
        patterns,
        pattern_consumers,
        consumers,
        exact_specs,
        alias_specs,
        lexical_specs,
        context_rules,
        conjunction_specs
    ))]
    fn new(
        patterns: Vec<String>,
        pattern_consumers: Vec<Vec<usize>>,
        consumers: Vec<(u8, usize, usize, usize)>,
        exact_specs: Vec<(String, String, u8, usize)>,
        alias_specs: Vec<(usize, i64, Option<i64>, i8, u8, bool, Option<usize>)>,
        lexical_specs: Vec<(usize, Vec<String>, bool, bool, bool, bool)>,
        context_rules: Vec<(usize, Vec<Vec<String>>, Vec<String>)>,
        conjunction_specs: Vec<(usize, usize, usize, usize)>,
    ) -> PyResult<Self> {
        if patterns.len() != pattern_consumers.len() {
            return Err(PyValueError::new_err("pattern_consumers length mismatch"));
        }
        let automaton = build_automaton(&patterns)?;
        let consumers = consumers
            .into_iter()
            .map(|(kind, a, b, c)| ConsumerSpec { kind, a, b, c })
            .collect();
        let mut parsed_exact = Vec::with_capacity(exact_specs.len());
        for (value, mode, kind, index) in exact_specs {
            if value.is_empty() {
                return Err(PyValueError::new_err("exact value must be non-empty"));
            }
            let mode = match mode.as_str() {
                "case_sensitive" => ExactMode::CaseSensitive,
                "python_ignorecase_ascii_literal" => {
                    if !value.is_ascii() {
                        return Err(PyValueError::new_err("ignorecase literal must be ASCII"));
                    }
                    ExactMode::PythonIgnoreCaseAsciiLiteral
                }
                _ => return Err(PyValueError::new_err(format!("unknown exact mode: {mode}"))),
            };
            parsed_exact.push(ExactSpec {
                value: value.chars().collect(),
                mode,
                kind,
                index,
            });
        }
        let aliases = alias_specs
            .into_iter()
            .map(
                |(
                    canonical_group,
                    valid_from_epoch,
                    valid_to_epoch,
                    family,
                    entity_kind,
                    unresolved_tag,
                    context_rule,
                )| AliasSpec {
                    canonical_group,
                    valid_from_epoch,
                    valid_to_epoch,
                    family,
                    entity_kind,
                    unresolved_tag,
                    context_rule,
                },
            )
            .collect();
        let lexical = lexical_specs
            .into_iter()
            .map(
                |(
                    context_window_chars,
                    terms,
                    is_event,
                    market_wide_authorized,
                    unresolved_tag,
                    has_event_family,
                )| LexicalSpec {
                    context_window_chars,
                    context_terms: terms.into_iter().map(String::into_bytes).collect(),
                    is_event,
                    market_wide_authorized,
                    unresolved_tag,
                    has_event_family,
                },
            )
            .collect();
        let contexts = context_rules
            .into_iter()
            .map(|(window_chars, clauses, rejects)| ContextRule {
                window_chars,
                accept_clauses: clauses
                    .into_iter()
                    .map(|clause| clause.into_iter().map(String::into_bytes).collect())
                    .collect(),
                reject_terms: rejects.into_iter().map(String::into_bytes).collect(),
            })
            .collect();
        let conjunctions = conjunction_specs
            .into_iter()
            .map(
                |(
                    actor_phrase_count,
                    action_phrase_count,
                    object_phrase_count,
                    scope_phrase_count,
                )| ConjunctionSpec {
                    actor_phrase_count,
                    action_phrase_count,
                    object_phrase_count,
                    scope_phrase_count,
                },
            )
            .collect();
        let exact_specs = ExactSpecs::new(parsed_exact)?;
        Ok(Self {
            automaton,
            pattern_consumers,
            consumers,
            exact_specs,
            aliases,
            lexical,
            contexts,
            conjunctions,
        })
    }

    #[pyo3(signature = (texts, epochs, workers=4))]
    fn screen_facts_batch(
        &self,
        py: Python<'_>,
        texts: Vec<String>,
        epochs: Vec<i64>,
        workers: usize,
    ) -> PyResult<Vec<FactRow>> {
        if texts.len() != epochs.len() {
            return Err(PyValueError::new_err("texts/epochs length mismatch"));
        }
        if workers == 0 {
            return Err(PyValueError::new_err("workers must be >= 1"));
        }
        let pool = ThreadPoolBuilder::new()
            .num_threads(workers)
            .thread_name(|index| format!("news-relevance-worker-{index}"))
            .build()
            .map_err(|error| PyValueError::new_err(error.to_string()))?;
        let automaton = &self.automaton;
        let pattern_consumers = &self.pattern_consumers;
        let consumers = &self.consumers;
        let exact_specs = &self.exact_specs;
        let aliases = &self.aliases;
        let lexical = &self.lexical;
        let contexts = &self.contexts;
        let conjunctions = &self.conjunctions;
        Ok(py.detach(move || {
            pool.install(|| {
                texts
                    .par_iter()
                    .zip(epochs.par_iter())
                    .enumerate()
                    .map_init(ScanScratch::default, |scratch, (index, (text, epoch))| {
                        scan_one(
                            automaton,
                            pattern_consumers,
                            consumers,
                            exact_specs,
                            aliases,
                            lexical,
                            contexts,
                            conjunctions,
                            text,
                            *epoch,
                            index,
                            scratch,
                        )
                    })
                    .filter_map(|row| row)
                    .collect()
            })
        }))
    }
}

#[cfg(test)]
mod matcher_tests {
    use super::*;

    fn spec(value: &str, mode: ExactMode, index: usize) -> ExactSpec {
        ExactSpec {
            value: value.chars().collect(),
            mode,
            kind: ALIAS,
            index,
        }
    }

    fn synthetic_patterns(count: usize) -> Vec<String> {
        (0..count)
            .map(|index| format!("synthetic pattern {index:04}"))
            .collect()
    }

    fn overlapping_matches(automaton: &AhoCorasick, text: &str) -> Vec<(usize, usize, usize)> {
        let mut matches: Vec<_> = automaton
            .find_overlapping_iter(text.as_bytes())
            .map(|matched| {
                (
                    matched.pattern().as_usize(),
                    matched.start(),
                    matched.end(),
                )
            })
            .collect();
        matches.sort_unstable();
        matches
    }

    #[test]
    fn default_sized_normalized_taxonomy_selects_bounded_dfa() {
        // The maintained taxonomy currently has 226 normalized patterns, so
        // this sits above aho-corasick's automatic DFA threshold while still
        // remaining comfortably inside RANGE's explicit safety bounds.
        let patterns = synthetic_patterns(226);
        assert!(normalized_dfa_eligible(&patterns));
        let automaton = build_automaton(&patterns).expect("bounded DFA builds");
        assert_eq!(automaton.kind(), AhoCorasickKind::DFA);
        assert!(automaton.memory_usage() <= NORMALIZED_DFA_MAX_HEAP_BYTES);
    }

    #[test]
    fn large_custom_taxonomy_falls_back_to_ordinary_auto_selection() {
        let patterns = synthetic_patterns(NORMALIZED_DFA_MAX_PATTERNS + 1);
        assert!(!normalized_dfa_eligible(&patterns));
        let selected = build_automaton(&patterns).expect("fallback automaton builds");
        let automatic = build_auto_automaton(&patterns, false).expect("auto automaton builds");
        assert_eq!(selected.kind(), automatic.kind());
        assert_ne!(selected.kind(), AhoCorasickKind::DFA);
    }

    #[test]
    fn bounded_dfa_preserves_standard_overlapping_matches() {
        let mut patterns = synthetic_patterns(101);
        patterns.extend([
            "equity market".to_string(),
            "equity market selloff".to_string(),
            "market selloff".to_string(),
            "capital rule".to_string(),
            "rule".to_string(),
        ]);
        let text = "a capital rule preceded an equity market selloff";

        let selected = build_automaton(&patterns).expect("bounded DFA builds");
        let automatic = build_auto_automaton(&patterns, false).expect("auto automaton builds");
        assert_eq!(selected.kind(), AhoCorasickKind::DFA);
        assert_ne!(automatic.kind(), AhoCorasickKind::DFA);
        assert_eq!(selected.match_kind(), MatchKind::Standard);
        assert_eq!(
            overlapping_matches(&selected, text),
            overlapping_matches(&automatic, text)
        );
    }

    #[test]
    fn exact_automata_equal_scalar_reference_across_boundaries_and_unicode() {
        let specs = vec![
            spec("UPRO", ExactMode::CaseSensitive, 0),
            spec("QQQ", ExactMode::PythonIgnoreCaseAsciiLiteral, 1),
            spec("iShares", ExactMode::PythonIgnoreCaseAsciiLiteral, 2),
            spec("PowerShares", ExactMode::PythonIgnoreCaseAsciiLiteral, 3),
            // Duplicate literals must still fan out to every configured consumer.
            spec("SEC", ExactMode::CaseSensitive, 4),
            spec("SEC", ExactMode::CaseSensitive, 5),
        ];
        let matcher = ExactSpecs::new(specs).expect("valid exact automata");
        let texts = [
            "UPRO upro QQQ qqq xQQQ QQQx SEC",
            "é UPRO αQQQ αSEC",
            "İShares ıSHARES iShares Powerſhares",
            "İQQQ QQQİ ſQQQ QQQK",
            "no exact literals here",
        ];

        for text in texts {
            let chars: Vec<char> = text.chars().collect();
            let mut expected = Vec::new();
            for (spec_index, spec) in matcher.specs.iter().enumerate() {
                expected.extend(
                    exact_matches(&chars, spec)
                        .into_iter()
                        .map(|(start, end)| (spec_index, start, end)),
                );
            }
            expected.sort_unstable();

            let mut observed = matcher.matching(text);
            observed.sort_unstable();
            assert_eq!(observed, expected, "text={text:?}");
            assert_eq!(!observed.is_empty(), !expected.is_empty());
        }
    }

    #[test]
    fn python_ignorecase_exceptions_remain_token_boundaries() {
        let matcher = ExactSpecs::new(vec![spec(
            "QQQ",
            ExactMode::PythonIgnoreCaseAsciiLiteral,
            0,
        )])
        .expect("valid exact automaton");
        for text in ["İQQQ", "QQQı", "ſQQQ", "QQQK"] {
            assert!(matcher.matching(text).is_empty(), "text={text:?}");
        }
    }

    #[test]
    fn direct_text_normalization_matches_character_slice_reference() {
        let texts = [
            "ASCII -- punctuation\tand spaces",
            "Café — ‘quoted’ ／ fullwidth",
            "Straße İ ı ſ K fi",
            "\u{001c}leading\u{00a0}and\u{2029}trailing\n",
            "",
        ];
        for text in texts {
            let chars: Vec<char> = text.chars().collect();
            let mut expected = Vec::new();
            let mut expected_offsets = Vec::new();
            normalize_chars_into(&chars, 0, &mut expected, &mut expected_offsets);

            let mut observed = Vec::new();
            let mut observed_offsets = Vec::new();
            let (all_ascii, exceptional_ignorecase, source_len) =
                normalize_text_into(text, &mut observed, &mut observed_offsets);
            assert_eq!(observed, expected, "text={text:?}");
            assert_eq!(observed_offsets, expected_offsets, "text={text:?}");
            assert_eq!(source_len, chars.len(), "text={text:?}");
            assert_eq!(
                all_ascii,
                chars.iter().all(|character| character.is_ascii())
            );
            assert_eq!(
                exceptional_ignorecase,
                chars.iter().any(|character| matches!(
                    character,
                    '\u{0130}' | '\u{0131}' | '\u{017f}' | '\u{212a}'
                )),
                "text={text:?}"
            );
        }
    }

    // Reuse all reference/candidate buffers across exhaustive synthetic cases.
    // This keeps auxiliary memory bounded by the longest individual fixture.
    fn normalization_reference_checker() -> impl FnMut(&str) {
        let mut chars = Vec::new();
        let mut expected = Vec::new();
        let mut expected_offsets = Vec::new();
        let mut observed = Vec::new();
        let mut observed_offsets = Vec::new();
        move |text: &str| {
            chars.clear();
            chars.extend(text.chars());
            normalize_chars_into(&chars, 0, &mut expected, &mut expected_offsets);
            // The existing exhaustive scalar and boundary fixtures explicitly
            // exercise both forced scalar and runtime-guarded vector dispatch.
            // On non-AVX2 machines both are portable scalar; vector status must
            // be reported as unexercised, not counted as a passing vector gate.
            for allow_avx2 in [false, true] {
                let flags = normalize_text_into_with_mixed_avx2(
                    text, &mut observed, &mut observed_offsets, allow_avx2,
                );
                assert_eq!(observed, expected, "allow_avx2={allow_avx2} text={text:?}");
                assert_eq!(observed_offsets, expected_offsets,
                           "allow_avx2={allow_avx2} text={text:?}");
                assert_eq!(
                    flags,
                    (
                        text.is_ascii(),
                        chars.iter().any(|character| matches!(
                            character,
                            '\u{0130}' | '\u{0131}' | '\u{017f}' | '\u{212a}'
                        )),
                        chars.len(),
                    ),
                    "allow_avx2={allow_avx2} text={text:?}",
                );
            }
        }
    }

    #[test]
    fn mixed_ascii_runs_match_reference_for_every_valid_scalar() {
        let mut check = normalization_reference_checker();
        // Reuse one bounded buffer. Each non-ASCII scalar has a complete
        // vector block on both sides; ASCII scalars occur inside a vector block.
        // NBSP keeps every scalar fixture on the mixed-row branch. Both forced
        // scalar and guarded vector modes run in the shared reference checker.
        let prefix = "A \t-B`/ ".repeat(4);
        let suffix = " /C D`B ".repeat(4);
        assert_eq!(prefix.len(), 32);
        assert_eq!(suffix.len(), 32);
        let mut text = String::with_capacity(prefix.len() + suffix.len() + 8);
        // One fixture per valid scalar, not a scalar/run-length Cartesian product.
        for codepoint in 0..=0x10ffff {
            let Some(character) = char::from_u32(codepoint) else {
                continue;
            };
            text.clear();
            text.push_str(&prefix);
            text.push(character);
            text.push_str(&suffix);
            text.push_str("\u{00a0}D");
            check(&text);
        }
    }

    #[test]
    fn mixed_ascii_runs_preserve_all_ascii_bytes_at_unicode_boundaries() {
        let mut check = normalization_reference_checker();
        let mut text = String::with_capacity(160);
        let unicode = [
            '\u{0080}', 'é', '\u{0301}', '—', '\u{00a0}', '…', '\u{2029}',
            'fi', 'İ', 'ı', 'ſ', 'K', '界', '🦀', '\u{fdfa}', '\u{10ffff}',
        ];
        for run_length in [0, 1, 15, 16, 31, 32, 33, 63, 64, 65, 127, 128, 129] {
            for byte in 0u8..=0x7f {
                for character in unicode {
                    text.clear();
                    text.extend(std::iter::repeat('A').take(run_length));
                    text.push(byte as char);
                    text.push(character);
                    text.push(byte as char);
                    text.push_str(" \tB/");
                    check(&text);

                    // Also cover a Unicode scalar first and last, with the
                    // same ASCII byte on both sides of a variable-length run.
                    text.clear();
                    text.push(character);
                    text.push(byte as char);
                    text.extend(std::iter::repeat('Z').take(run_length));
                    text.push(byte as char);
                    text.push(character);
                    check(&text);
                }
            }
        }
    }

    #[test]
    fn mixed_ascii_runs_keep_pending_space_and_expansion_source_offsets() {
        let mut normalized = Vec::new();
        let mut offsets = Vec::new();
        assert_eq!(
            normalize_text_into("\u{00a0}\u{001c}É — \tB", &mut normalized, &mut offsets),
            (false, false, 8),
        );
        assert_eq!(normalized, "é b".as_bytes());
        assert_eq!(offsets, [2, 2, 7, 7]);
        assert_eq!(
            normalize_text_into("A\u{00a0}fi\u{001c}B", &mut normalized, &mut offsets),
            (false, false, 5),
        );
        assert_eq!(normalized, b"a fi b");
        assert_eq!(offsets, [0, 2, 2, 2, 4, 4]);
        let mut check = normalization_reference_checker();
        for text in [
            "éα界🦀", "İIıſK", "\u{00a0}\u{2029}—", "A\u{0301}B",
            "\u{0301}A\u{0301}", "A—\u{00a0}\u{001c}B", "A\u{fdfa}B",
        ] {
            check(text);
        }
    }

    fn assert_ascii_normalization_matches_character_reference(bytes: &[u8]) {
        let text = std::str::from_utf8(bytes).expect("ASCII bytes are valid UTF-8");
        let chars: Vec<char> = text.chars().collect();
        let mut expected = Vec::new();
        let mut expected_offsets = Vec::new();
        normalize_chars_into(&chars, 0, &mut expected, &mut expected_offsets);

        let mut observed = Vec::new();
        let mut observed_offsets = Vec::new();
        let flags = normalize_text_into(text, &mut observed, &mut observed_offsets);
        assert_eq!(flags, (true, false, bytes.len()), "bytes={bytes:?}");
        assert_eq!(observed, expected, "bytes={bytes:?}");
        assert_eq!(observed_offsets, expected_offsets, "bytes={bytes:?}");
    }

    #[test]
    fn ascii_byte_path_matches_character_reference_exhaustively() {
        assert_ascii_normalization_matches_character_reference(b"");
        for first in 0u8..=0x7f {
            assert_ascii_normalization_matches_character_reference(&[first]);
            assert_ascii_normalization_matches_character_reference(&[b'x', first, b'y']);
            for second in 0u8..=0x7f {
                assert_ascii_normalization_matches_character_reference(&[first, second]);
                assert_ascii_normalization_matches_character_reference(&[
                    b'x', first, second, b'y',
                ]);
            }
        }
    }

    #[test]
    fn ascii_byte_path_preserves_collapse_and_control_semantics() {
        let tricky = [
            b' ', b'-', b'\t', b'A', b'`', b'B', b'.', b'/', b' ', 0x00, b'\\', 0x7f, b' ', b'\n',
        ];
        assert_ascii_normalization_matches_character_reference(&tricky);

        let mut normalized = Vec::new();
        let mut offsets = Vec::new();
        let text = std::str::from_utf8(&tricky).expect("ASCII bytes are valid UTF-8");
        assert_eq!(
            normalize_text_into(text, &mut normalized, &mut offsets),
            (true, false, tricky.len())
        );
        assert_eq!(normalized, b"a'b \0\\\x7f");
        assert_eq!(offsets, [3, 4, 5, 9, 9, 10, 11]);

        let python_ascii_whitespace = [
            b' ', b'\t', b'\n', 0x0b, 0x0c, b'\r', 0x1c, 0x1d, 0x1e, 0x1f,
        ];
        let separators = [
            b'-', b'.', b',', b':', b';', b'(', b')', b'[', b']', b'{', b'}', b'/',
        ];
        let mut runs = vec![b'A'];
        runs.extend_from_slice(&python_ascii_whitespace);
        runs.extend_from_slice(&separators);
        runs.push(b'Z');
        assert_ascii_normalization_matches_character_reference(&runs);
        normalize_text_into(
            std::str::from_utf8(&runs).expect("ASCII bytes are valid UTF-8"),
            &mut normalized,
            &mut offsets,
        );
        assert_eq!(normalized, b"a z");
        assert_eq!(offsets, [0, runs.len() - 1, runs.len() - 1]);
    }

    #[test]
    fn borrowed_context_windows_match_isolated_normalization_exhaustively() {
        let texts = [
            "",
            "a",
            "  --Federal\tReserve...cut rates  ",
            "prefix---GDP\u{00a0}growth／suffix",
            "Café — ‘SEC’ ／ filing",
            "İ ı ſ K fi — SPY ETF",
            "x\u{001c}\u{001d}market\u{2029}structure\n",
        ];

        for text in texts {
            let chars: Vec<char> = text.chars().collect();
            let mut normalized = Vec::new();
            let mut offsets = Vec::new();
            let (_, _, source_len) = normalize_text_into(text, &mut normalized, &mut offsets);
            assert_eq!(source_len, chars.len());

            for start in 0..=source_len {
                for end in start..=source_len {
                    for width in [0, 1, 2, 4, source_len.saturating_add(2)] {
                        let left = start.saturating_sub(width);
                        let right = end.saturating_add(width).min(source_len);
                        let mut expected = Vec::new();
                        let mut expected_offsets = Vec::new();
                        normalize_chars_into(
                            &chars[left..right],
                            left,
                            &mut expected,
                            &mut expected_offsets,
                        );
                        let observed = normalized_context_window(
                            &normalized,
                            &offsets,
                            source_len,
                            start,
                            end,
                            width,
                        );
                        assert_eq!(
                            observed, expected,
                            "text={text:?}, start={start}, end={end}, width={width}"
                        );
                    }
                }
            }
        }
    }

    #[test]
    fn borrowed_context_windows_preserve_phrase_boundaries() {
        let text = "prefederal -- Federal Reserve; GDP-growth; SECond SEC filing; SPY ETF";
        let chars: Vec<char> = text.chars().collect();
        let mut normalized = Vec::new();
        let mut offsets = Vec::new();
        let (_, _, source_len) = normalize_text_into(text, &mut normalized, &mut offsets);
        let terms: [&[u8]; 7] = [
            b"fed",
            b"federal reserve",
            b"gdp",
            b"growth",
            b"sec",
            b"spy",
            b"etf",
        ];

        for start in 0..=source_len {
            for end in start..=source_len {
                for width in [0, 1, 8, source_len.saturating_add(1)] {
                    let left = start.saturating_sub(width);
                    let right = end.saturating_add(width).min(source_len);
                    let mut expected = Vec::new();
                    let mut expected_offsets = Vec::new();
                    normalize_chars_into(
                        &chars[left..right],
                        left,
                        &mut expected,
                        &mut expected_offsets,
                    );
                    let observed = normalized_context_window(
                        &normalized,
                        &offsets,
                        source_len,
                        start,
                        end,
                        width,
                    );
                    for term in terms {
                        assert_eq!(
                            normalized_contains(observed, term),
                            normalized_contains(&expected, term),
                            "term={term:?}, start={start}, end={end}, width={width}"
                        );
                    }
                }
            }
        }
    }

    #[test]
    fn custom_spy_context_keeps_reject_precedence() {
        let automaton =
            build_automaton(&["never matches".to_string()]).expect("valid normalized automaton");
        let exact_specs = ExactSpecs::new(vec![ExactSpec {
            value: "SPY".chars().collect(),
            mode: ExactMode::CaseSensitive,
            kind: ALIAS,
            index: 0,
        }])
        .expect("valid exact alias");
        let aliases = vec![AliasSpec {
            canonical_group: 0,
            context_rule: Some(0),
            valid_from_epoch: 0,
            valid_to_epoch: None,
            family: FAMILY_SP500,
            entity_kind: 0,
            unresolved_tag: false,
        }];
        let contexts = vec![ContextRule {
            window_chars: 128,
            accept_clauses: vec![
                vec![b"etf".to_vec()],
                vec![b"fund".to_vec(), b"s&p 500".to_vec()],
            ],
            reject_terms: vec![b"spy plane".to_vec(), b"espionage".to_vec()],
        }];
        let mut scratch = ScanScratch::default();
        let scan = |text: &str, row_index: usize, scratch: &mut ScanScratch| {
            scan_one(
                &automaton,
                &[Vec::new()],
                &[],
                &exact_specs,
                &aliases,
                &[],
                &contexts,
                &[],
                text,
                1,
                row_index,
                scratch,
            )
            .expect("SPY remains an observed alias")
        };

        let accepted = scan("SPY — ETF shares rose", 0, &mut scratch);
        assert_eq!(accepted.1, ARTICLE_ACCEPTED);
        assert_eq!(accepted.2, EXP_SUPPORTED);

        let rejected = scan("SPY ETF appeared beside a spy plane", 1, &mut scratch);
        assert_eq!(rejected.1, ARTICLE_REJECTED);
        assert_eq!(rejected.2, EXP_NOT_SUPPORTED);

        let missing = scan("SPY reported quarterly earnings", 2, &mut scratch);
        assert_eq!(missing.1, ARTICLE_REJECTED);
        assert_eq!(missing.2, EXP_NOT_SUPPORTED);
    }

    #[test]
    fn representative_fed_gdp_and_sec_contexts_keep_frozen_behavior() {
        let automaton =
            build_automaton(&["federal reserve".to_string()]).expect("valid normalized automaton");
        let pattern_consumers = vec![vec![0]];
        let consumers = vec![ConsumerSpec {
            kind: LEXICAL,
            a: 0,
            b: 0,
            c: 0,
        }];
        let exact_specs = ExactSpecs::new(vec![
            ExactSpec {
                value: "GDP".chars().collect(),
                mode: ExactMode::CaseSensitive,
                kind: LEXICAL,
                index: 1,
            },
            ExactSpec {
                value: "SEC".chars().collect(),
                mode: ExactMode::CaseSensitive,
                kind: LEXICAL,
                index: 2,
            },
        ])
        .expect("valid exact lexical matchers");
        let lexical = vec![
            LexicalSpec {
                context_window_chars: 0,
                context_terms: Vec::new(),
                is_event: false,
                market_wide_authorized: true,
                unresolved_tag: false,
                has_event_family: false,
            },
            LexicalSpec {
                context_window_chars: 64,
                context_terms: vec![b"growth".to_vec(), b"economy".to_vec()],
                is_event: false,
                market_wide_authorized: true,
                unresolved_tag: false,
                has_event_family: false,
            },
            LexicalSpec {
                context_window_chars: 96,
                context_terms: vec![b"commission".to_vec(), b"filing".to_vec()],
                is_event: false,
                market_wide_authorized: false,
                unresolved_tag: false,
                has_event_family: false,
            },
        ];
        let mut scratch = ScanScratch::default();
        let scan = |text: &str, row_index: usize, scratch: &mut ScanScratch| {
            scan_one(
                &automaton,
                &pattern_consumers,
                &consumers,
                &exact_specs,
                &[],
                &lexical,
                &[],
                &[],
                text,
                1,
                row_index,
                scratch,
            )
        };

        let fed = scan("The Federal Reserve cut rates", 0, &mut scratch)
            .expect("Federal Reserve is relevant");
        assert_eq!(fed.1, ARTICLE_ACCEPTED);
        assert_eq!(fed.5, EXP_SUPPORTED);

        let gdp = scan("Economic growth lifted GDP", 1, &mut scratch)
            .expect("contextual GDP is observed");
        assert_eq!(gdp.1, ARTICLE_ACCEPTED);
        assert_eq!(gdp.5, EXP_SUPPORTED);

        let gdp_without_context = scan("The GDP widget was renamed", 2, &mut scratch)
            .expect("GDP remains an observation when context fails");
        assert_eq!(gdp_without_context.1, ARTICLE_REJECTED);
        assert_eq!(gdp_without_context.5, EXP_NOT_SUPPORTED);

        let sec = scan("The SEC filing was published", 3, &mut scratch)
            .expect("contextual SEC is observed");
        assert_eq!(sec.1, ARTICLE_ACCEPTED);
        assert_eq!(sec.5, EXP_NOT_SUPPORTED);
        assert!(scan("SECond place", 4, &mut scratch).is_none());
    }

    #[test]
    fn reused_scan_scratch_does_not_leak_candidates_between_rows() {
        let automaton = build_automaton(&["alpha".to_string()]).expect("valid automaton");
        let pattern_consumers = vec![vec![0]];
        let consumers = vec![ConsumerSpec {
            kind: ALIAS,
            a: 0,
            b: 0,
            c: 0,
        }];
        let exact_specs = ExactSpecs::new(Vec::new()).expect("empty exact specs are valid");
        let aliases = vec![AliasSpec {
            canonical_group: 0,
            context_rule: None,
            valid_from_epoch: 0,
            valid_to_epoch: None,
            family: FAMILY_SP500,
            entity_kind: 0,
            unresolved_tag: false,
        }];
        let mut scratch = ScanScratch::default();
        let scan = |text: &str, row_index: usize, scratch: &mut ScanScratch| {
            scan_one(
                &automaton,
                &pattern_consumers,
                &consumers,
                &exact_specs,
                &aliases,
                &[],
                &[],
                &[],
                text,
                1,
                row_index,
                scratch,
            )
        };

        let first = scan("Alpha rises", 0, &mut scratch).expect("first row matches");
        assert_eq!(first.0, 0);
        assert_eq!(first.6, 1);
        assert!(scan("nothing relevant", 1, &mut scratch).is_none());
        let third = scan("alpha again", 2, &mut scratch).expect("third row matches");
        assert_eq!(third.0, 2);
        assert_eq!(third.6, 1);
    }

    #[test]
    fn conjunction_marks_resize_reset_and_survive_generation_wrap() {
        let mut marks = ConjunctionMarks::default();
        marks.begin_row(130);
        for field in 0..4 {
            marks.mark(129, field);
        }
        assert_eq!(marks.touched, [129]);
        assert!(marks.is_complete(129));

        marks.begin_row(130);
        assert!(marks.is_empty());
        assert!(!marks.is_complete(129));

        marks.generation = u32::MAX;
        marks.begin_row(130);
        assert_eq!(marks.generation, 1);
        assert!(marks.stamps.iter().all(|stamp| *stamp == 0));
        assert!(!marks.is_complete(129));
    }

    #[test]
    fn sorted_alias_candidates_preserve_longest_ties_and_frozen_counting() {
        let automaton =
            build_automaton(&["alpha".to_string(), "alpha beta".to_string()])
                .expect("valid automaton");
        let pattern_consumers = vec![vec![0], vec![1, 1, 2]];
        let consumers = vec![
            ConsumerSpec {
                kind: ALIAS,
                a: 0,
                b: 0,
                c: 0,
            },
            ConsumerSpec {
                kind: ALIAS,
                a: 1,
                b: 0,
                c: 0,
            },
            ConsumerSpec {
                kind: ALIAS,
                a: 2,
                b: 0,
                c: 0,
            },
        ];
        let aliases = vec![
            AliasSpec {
                canonical_group: 0,
                context_rule: None,
                valid_from_epoch: 0,
                valid_to_epoch: None,
                family: FAMILY_SP500,
                entity_kind: 0,
                unresolved_tag: false,
            },
            AliasSpec {
                canonical_group: 0,
                context_rule: None,
                valid_from_epoch: 0,
                valid_to_epoch: None,
                family: FAMILY_SP500,
                entity_kind: 0,
                unresolved_tag: false,
            },
            // Retained observations are counted before temporal rejection.
            AliasSpec {
                canonical_group: 0,
                context_rule: None,
                valid_from_epoch: 10,
                valid_to_epoch: None,
                family: FAMILY_SP500,
                entity_kind: 0,
                unresolved_tag: false,
            },
        ];
        let exact_specs = ExactSpecs::new(Vec::new()).expect("empty exact specs are valid");
        let mut scratch = ScanScratch::default();
        let observed = scan_one(
            &automaton,
            &pattern_consumers,
            &consumers,
            &exact_specs,
            &aliases,
            &[],
            &[],
            &[],
            "alpha beta",
            1,
            0,
            &mut scratch,
        )
        .expect("long aliases match");

        assert_eq!(observed.1, ARTICLE_ACCEPTED);
        assert_eq!(observed.2, EXP_SUPPORTED);
        assert_eq!(observed.6, 2);
    }

    #[test]
    fn sorted_lexical_candidates_deduplicate_per_rule_and_span() {
        let automaton = build_automaton(&["signal".to_string()]).expect("valid automaton");
        let pattern_consumers = vec![vec![0, 0]];
        let consumers = vec![ConsumerSpec {
            kind: LEXICAL,
            a: 0,
            b: 0,
            c: 0,
        }];
        let lexical = vec![LexicalSpec {
            context_window_chars: 0,
            context_terms: Vec::new(),
            is_event: true,
            market_wide_authorized: false,
            unresolved_tag: false,
            has_event_family: true,
        }];
        let exact_specs = ExactSpecs::new(Vec::new()).expect("empty exact specs are valid");
        let mut scratch = ScanScratch::default();
        let observed = scan_one(
            &automaton,
            &pattern_consumers,
            &consumers,
            &exact_specs,
            &[],
            &lexical,
            &[],
            &[],
            "signal and signal",
            1,
            0,
            &mut scratch,
        )
        .expect("lexical events match");

        assert_eq!(observed.1, ARTICLE_ACCEPTED);
        assert_eq!(observed.6, 2);
    }

    #[test]
    fn conjunction_masks_count_once_and_do_not_leak_between_rows() {
        let automaton = build_automaton(&[
            "commission".to_string(),
            "adopted".to_string(),
            "rule".to_string(),
            "market".to_string(),
        ])
        .expect("valid automaton");
        let pattern_consumers = vec![vec![0, 0], vec![1, 1], vec![2], vec![3]];
        let consumers = vec![
            ConsumerSpec {
                kind: CONJUNCTION,
                a: 0,
                b: 0,
                c: 0,
            },
            ConsumerSpec {
                kind: CONJUNCTION,
                a: 0,
                b: 1,
                c: 0,
            },
            ConsumerSpec {
                kind: CONJUNCTION,
                a: 0,
                b: 2,
                c: 0,
            },
            ConsumerSpec {
                kind: CONJUNCTION,
                a: 0,
                b: 3,
                c: 0,
            },
        ];
        let exact_specs = ExactSpecs::new(vec![ExactSpec {
            value: "SEC".chars().collect(),
            mode: ExactMode::CaseSensitive,
            kind: CONJUNCTION,
            index: 0,
        }])
        .expect("valid exact actor");
        let conjunctions = vec![ConjunctionSpec {
            actor_phrase_count: 1,
            action_phrase_count: 1,
            object_phrase_count: 1,
            scope_phrase_count: 1,
        }];
        let mut scratch = ScanScratch::default();
        let scan = |text: &str, row_index: usize, scratch: &mut ScanScratch| {
            scan_one(
                &automaton,
                &pattern_consumers,
                &consumers,
                &exact_specs,
                &[],
                &[],
                &[],
                &conjunctions,
                text,
                1,
                row_index,
                scratch,
            )
        };

        let first = scan(
            "SEC commission adopted adopted rule market",
            0,
            &mut scratch,
        )
        .expect("complete conjunction");
        assert_eq!(first.6, 1);
        assert_eq!(first.5, EXP_SUPPORTED);

        assert!(scan("SEC adopted rule", 1, &mut scratch).is_none());
        let third = scan("commission adopted rule market", 2, &mut scratch)
            .expect("later complete conjunction");
        assert_eq!(third.6, 1);
    }
    // Block fixtures. Existing normalization_reference_checker is also
    // changed by the proposal to run each current scalar fixture in both modes.
    #[cfg(target_arch = "x86_64")]
    #[test]
    fn mixed_avx2_prefix_stops_at_original_unicode_boundaries() {
        println!("Phase31 block runtime AVX2 available: {}", mixed_ascii_avx2_available());
        if !mixed_ascii_avx2_available() {
            return; // The portable/forced-scalar fixtures still run everywhere.
        }
        let pattern = b"A \t-`B/\0\x1c\x7f.,:;()[]{}";
        for prefix_len in 0..=96 {
            for character in ['é', 'fi', 'İ', 'ı', 'ſ', 'K', '界', '🦀', '\u{fdfa}'] {
                let prefix: Vec<u8> = pattern.iter().copied().cycle().take(prefix_len).collect();
                let mut text = String::from_utf8(prefix.clone()).unwrap();
                text.push(character);
                text.push_str("ABCDEFGHIJKLMNOPQRSTUVWXYZ 0123456789");
                for carry_space in [false, true] {
                    // Nonzero base proves offsets are source positions, not
                    // local block positions. A previous expansion is retained.
                    let mut expected = b"fi".to_vec();
                    let mut expected_offsets = vec![5, 5];
                    let mut expected_pending = carry_space;
                    for (index, byte) in prefix.iter().copied().enumerate() {
                        normalize_source_char(byte as char, 9 + index, &mut expected,
                                              &mut expected_offsets, &mut expected_pending);
                    }
                    let mut observed = b"fi".to_vec();
                    let mut observed_offsets = vec![5, 5];
                    let mut observed_pending = carry_space;
                    // SAFETY: runtime AVX2 check above; helper bounds every load.
                    let consumed = unsafe {
                        normalize_mixed_ascii_prefix_avx2_into(
                            text.as_bytes(), 9, &mut observed, &mut observed_offsets,
                            &mut observed_pending,
                        )
                    };
                    assert_eq!(consumed, prefix_len, "prefix={prefix_len} char={character:?}");
                    assert_eq!(observed, expected);
                    assert_eq!(observed_offsets, expected_offsets);
                    assert_eq!(observed_pending, expected_pending);
                }
            }
        }

        // Every ASCII byte in every SIMD lane, including whitespace and
        // separators inside blocks. This does not rely on whitespace-free data.
        for lane in 0..96 {
            for byte in 0u8..=0x7f {
                let mut prefix = vec![b'x'; 96];
                prefix[lane] = byte;
                let mut text = String::from_utf8(prefix.clone()).unwrap();
                text.push_str("fiABCDEFGHIJKLMNOPQRSTUVWXYZ 0123456789");
                let mut expected = Vec::new();
                let mut expected_offsets = Vec::new();
                let mut expected_pending = false;
                for (index, source_byte) in prefix.iter().copied().enumerate() {
                    normalize_source_char(source_byte as char, 7 + index, &mut expected,
                                          &mut expected_offsets, &mut expected_pending);
                }
                let mut observed = Vec::new();
                let mut observed_offsets = Vec::new();
                let mut observed_pending = false;
                let consumed = unsafe {
                    normalize_mixed_ascii_prefix_avx2_into(
                        text.as_bytes(), 7, &mut observed, &mut observed_offsets,
                        &mut observed_pending,
                    )
                };
                assert_eq!(consumed, 96);
                assert_eq!(observed, expected, "byte={byte} lane={lane}");
                assert_eq!(observed_offsets, expected_offsets, "byte={byte} lane={lane}");
                assert_eq!(observed_pending, expected_pending, "byte={byte} lane={lane}");
            }
        }

        // No padded/overreading load on short tails. Complete mixed fixtures
        // exercise the caller's scalar processing of these untouched bytes.
        for length in [0, 1, 15, 16, 31, 32, 33, 63, 64, 65] {
            let input = vec![b'A'; length];
            let mut normalized = Vec::new();
            let mut offsets = Vec::new();
            let mut pending_space = false;
            let consumed = unsafe {
                normalize_mixed_ascii_prefix_avx2_into(
                    &input, 7, &mut normalized, &mut offsets, &mut pending_space,
                )
            };
            assert_eq!(consumed, (length / 32) * 32);
            assert_eq!(normalized, vec![b'a'; consumed]);
            assert_eq!(offsets, (7..7 + consumed).collect::<Vec<_>>());
        }
    }

    #[test]
    fn mixed_block_modes_collapse_whole_blocks_across_expansions() {
        let mut check = normalization_reference_checker();
        for block_len in [32, 64] {
            // Every lane belongs to the collapse mask, including across two
            // consecutive blocks. Include spaces, controls and punctuation.
            let collapse: String = b" \t\n\r\x0b\x0c\x1c\x1d\x1e\x1f-.,:;()[]{}/"
                .iter().copied().cycle().take(block_len).map(char::from).collect();
            for text in [
                // No previous output: all-one masks must not create leading space.
                format!("{collapse}B\u{00a0}"),
                // Carry space through one/two masks after a Unicode expansion.
                format!("fi{collapse}B"),
                // Trailing masks must remain pending, never emitted at end-of-row.
                format!("fi{collapse}"),
                // A scalar Unicode space starts the same pending state before
                // the first vector block; repeated blocks must not duplicate it.
                format!("fi\u{00a0}{collapse}B"),
            ] {
                check(&text);
            }

            for allow_avx2 in [false, true] {
                let mut normalized = Vec::new();
                let mut offsets = Vec::new();
                let text = format!("fi{collapse}B");
                let flags = normalize_text_into_with_mixed_avx2(
                    &text, &mut normalized, &mut offsets, allow_avx2,
                );
                assert_eq!(flags, (false, false, block_len + 2));
                assert_eq!(normalized, b"fi b");
                assert_eq!(offsets, [0, 0, block_len + 1, block_len + 1]);

                let text = format!("fi{collapse}");
                normalize_text_into_with_mixed_avx2(
                    &text, &mut normalized, &mut offsets, allow_avx2,
                );
                assert_eq!(normalized, b"fi");
                assert_eq!(offsets, [0, 0]);
            }
        }
    }

    #[test]
    fn mixed_block_modes_keep_context_windows_across_vector_edges() {
        let texts = [
            format!("{}fi --\tİ{}\u{00a0}B", "A".repeat(31), "Z".repeat(33)),
            format!("\u{00a0}{}—{}\u{fdfa} ", "B`/ ".repeat(16), "C".repeat(31)),
            format!("{}Kıſ{}🦀{}", "A".repeat(32), "D".repeat(32), "E".repeat(33)),
        ];
        for text in texts {
            let chars: Vec<char> = text.chars().collect();
            for allow_avx2 in [false, true] {
                let mut normalized = Vec::new();
                let mut offsets = Vec::new();
                let flags = normalize_text_into_with_mixed_avx2(
                    &text, &mut normalized, &mut offsets, allow_avx2,
                );
                assert_eq!(flags.2, chars.len());
                for start in [0, 1, 15, 16, 31, 32, 33, 63, 64, chars.len()] {
                    let start = start.min(chars.len());
                    for span in [0, 1, 7] {
                        let end = (start + span).min(chars.len());
                        for width in [0, 1, 2, 31, 32, 33, 65] {
                            let left = start.saturating_sub(width);
                            let right = end.saturating_add(width).min(chars.len());
                            let mut expected = Vec::new();
                            let mut expected_offsets = Vec::new();
                            normalize_chars_into(&chars[left..right], left, &mut expected,
                                                 &mut expected_offsets);
                            assert_eq!(
                                normalized_context_window(&normalized, &offsets, chars.len(),
                                                          start, end, width),
                                expected.as_slice(),
                                "allow_avx2={allow_avx2} start={start} end={end} width={width}",
                            );
                        }
                    }
                }
            }
        }
    }

}