Spaces:
Running on Zero
Running on Zero
| //! Domain-neutral document gates sharing the frozen native text preparation. | |
| use super::*; | |
| type Hit = (usize, bool, usize, usize); | |
| type Row = (Vec<usize>, Vec<Hit>, bool); | |
| pub struct NativeCurationScanner { | |
| matcher: AhoCorasick, | |
| consumers: Vec<(usize, bool)>, | |
| rules: usize, | |
| patterns: Vec<String>, | |
| } | |
| impl NativeCurationScanner { | |
| fn new(rules: Vec<(Vec<String>, Vec<String>)>) -> PyResult<Self> { | |
| if rules.is_empty() { | |
| return Err(PyValueError::new_err("at least one rule required")); | |
| } | |
| let mut patterns = Vec::new(); | |
| let mut consumers = Vec::new(); | |
| for (id, (include, exclude)) in rules.iter().enumerate() { | |
| if include.is_empty() { | |
| return Err(PyValueError::new_err("include cannot be empty")); | |
| } | |
| for (negative, phrases) in [(false, include), (true, exclude)] { | |
| for phrase in phrases { | |
| let mut bytes = Vec::new(); | |
| let mut offsets = Vec::new(); | |
| normalize_text_into(phrase, &mut bytes, &mut offsets); | |
| if bytes.is_empty() { | |
| return Err(PyValueError::new_err("phrase normalizes to empty text")); | |
| } | |
| patterns.push(String::from_utf8(bytes).expect("normalizer produces UTF8")); | |
| consumers.push((id, negative)); | |
| } | |
| } | |
| } | |
| Ok(Self { | |
| matcher: build_auto_automaton(&patterns, false)?, | |
| consumers, | |
| rules: rules.len(), | |
| patterns, | |
| }) | |
| } | |
| /// Bounded, single-threaded matching-only comparison for the hosted demo. | |
| /// Preparation, sorting/verification and Python conversion are outside timers. | |
| fn compare_matching(&self, py: Python<'_>, texts: Vec<String>) -> PyResult<(f64, f64, usize)> { | |
| if texts.len() > 100 || texts.iter().map(|s| s.len()).sum::<usize>() > 800_000 | |
| || self.patterns.len() > 256 { | |
| return Err(PyValueError::new_err("matching comparison input limit exceeded")); | |
| } | |
| py.detach(|| { | |
| let prepared: Vec<String> = texts.iter().map(|text| { | |
| let (mut bytes, mut offsets) = (Vec::new(), Vec::new()); | |
| normalize_text_into(text, &mut bytes, &mut offsets); | |
| String::from_utf8(bytes).expect("normalizer produces UTF8") | |
| }).collect(); | |
| type Match = (usize, usize, usize, usize); | |
| let scan = |basic: bool| -> PyResult<Vec<Match>> { | |
| let mut matches = Vec::new(); | |
| for (row, text) in prepared.iter().enumerate() { | |
| let bytes = text.as_bytes(); | |
| let mut record = |pattern: usize, start: usize, end: usize| -> PyResult<()> { | |
| if (start == 0 || !is_token_byte(bytes[start - 1])) | |
| && (end == bytes.len() || !is_token_byte(bytes[end])) { | |
| if matches.len() == 100_000 { | |
| return Err(PyValueError::new_err("matching comparison hit limit exceeded")); | |
| } | |
| matches.push((row, pattern, start, end)); | |
| } | |
| Ok(()) | |
| }; | |
| if basic { | |
| for (pattern, phrase) in self.patterns.iter().enumerate() { | |
| let mut offset = 0; | |
| while let Some(relative) = text[offset..].find(phrase.as_str()) { | |
| let start = offset + relative; | |
| record(pattern, start, start + phrase.len())?; | |
| // Advance one character to retain overlapping matches. | |
| offset = start + text[start..].chars().next().unwrap().len_utf8(); | |
| } | |
| } | |
| } else { | |
| for m in self.matcher.find_overlapping_iter(bytes) { | |
| record(m.pattern().as_usize(), m.start(), m.end())?; | |
| } | |
| } | |
| } | |
| Ok(matches) | |
| }; | |
| let mut expected = scan(true)?; | |
| let mut actual = scan(false)?; | |
| expected.sort_unstable(); | |
| actual.sort_unstable(); | |
| if expected != actual { | |
| return Err(PyValueError::new_err("matching comparison outputs differ")); | |
| } | |
| let count = actual.len(); | |
| let (mut range_times, mut basic_times) = (Vec::new(), Vec::new()); | |
| for round in 0..5 { | |
| for basic in if round % 2 == 0 { [false, true] } else { [true, false] } { | |
| let started = std::time::Instant::now(); | |
| let hits = scan(basic)?; | |
| std::hint::black_box(&hits); | |
| let seconds = started.elapsed().as_secs_f64(); | |
| if basic { basic_times.push(seconds); } else { range_times.push(seconds); } | |
| } | |
| } | |
| range_times.sort_by(f64::total_cmp); | |
| basic_times.sort_by(f64::total_cmp); | |
| Ok((range_times[2], basic_times[2], count)) | |
| }) | |
| } | |
| fn scan( | |
| &self, | |
| py: Python<'_>, | |
| texts: Vec<String>, | |
| workers: usize, | |
| max_evidence: usize, | |
| ) -> PyResult<Vec<Row>> { | |
| let pool = worker_pool(workers)?; | |
| Ok(py.detach(|| { | |
| pool.install(|| { | |
| texts | |
| .par_iter() | |
| .map_init( | |
| || (Vec::new(), Vec::new(), vec![[false; 2]; self.rules]), | |
| |(normalized, offsets, found), text| { | |
| normalize_text_into(text, normalized, offsets); | |
| found.fill([false; 2]); | |
| let mut hits = Vec::new(); | |
| let mut truncated = false; | |
| for m in self.matcher.find_overlapping_iter(normalized.as_slice()) { | |
| let (start, end) = (m.start(), m.end()); | |
| if (start > 0 && is_token_byte(normalized[start - 1])) | |
| || (end < normalized.len() && is_token_byte(normalized[end])) | |
| { | |
| continue; | |
| } | |
| let (rule, negative) = self.consumers[m.pattern().as_usize()]; | |
| found[rule][usize::from(negative)] = true; | |
| if hits.len() < max_evidence { | |
| hits.push(( | |
| rule, | |
| negative, | |
| offsets[start], | |
| offsets[end - 1] + 1, | |
| )); | |
| } else if max_evidence > 0 { | |
| truncated = true; | |
| } | |
| } | |
| let labels = found | |
| .iter() | |
| .enumerate() | |
| .filter_map(|(i, f)| (f[0] && !f[1]).then_some(i)) | |
| .collect(); | |
| (labels, hits, truncated) | |
| }, | |
| ) | |
| .collect() | |
| }) | |
| })) | |
| } | |
| } | |