Spaces:
Running on Zero
Running on Zero
File size: 7,708 Bytes
2444ad5 c136210 2444ad5 c4aee46 2444ad5 c4aee46 2444ad5 c4aee46 2444ad5 c4aee46 2444ad5 c4aee46 c136210 c4aee46 2444ad5 c4aee46 2444ad5 c4aee46 2444ad5 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 | //! Domain-neutral document gates sharing the frozen native text preparation.
use super::*;
type Hit = (usize, bool, usize, usize);
type Row = (Vec<usize>, Vec<Hit>, bool);
#[pyclass]
pub struct NativeCurationScanner {
matcher: AhoCorasick,
consumers: Vec<(usize, bool)>,
rules: usize,
patterns: Vec<String>,
}
#[pymethods]
impl NativeCurationScanner {
#[new]
fn new(rules: Vec<(Vec<String>, Vec<String>)>) -> PyResult<Self> {
if rules.is_empty() {
return Err(PyValueError::new_err("at least one rule required"));
}
let mut patterns = Vec::new();
let mut consumers = Vec::new();
for (id, (include, exclude)) in rules.iter().enumerate() {
if include.is_empty() {
return Err(PyValueError::new_err("include cannot be empty"));
}
for (negative, phrases) in [(false, include), (true, exclude)] {
for phrase in phrases {
let mut bytes = Vec::new();
let mut offsets = Vec::new();
normalize_text_into(phrase, &mut bytes, &mut offsets);
if bytes.is_empty() {
return Err(PyValueError::new_err("phrase normalizes to empty text"));
}
patterns.push(String::from_utf8(bytes).expect("normalizer produces UTF8"));
consumers.push((id, negative));
}
}
}
Ok(Self {
matcher: build_auto_automaton(&patterns, false)?,
consumers,
rules: rules.len(),
patterns,
})
}
/// Bounded, single-threaded matching-only comparison for the hosted demo.
/// Preparation, sorting/verification and Python conversion are outside timers.
fn compare_matching(&self, py: Python<'_>, texts: Vec<String>) -> PyResult<(f64, f64, usize)> {
if texts.len() > 100 || texts.iter().map(|s| s.len()).sum::<usize>() > 800_000
|| self.patterns.len() > 256 {
return Err(PyValueError::new_err("matching comparison input limit exceeded"));
}
py.detach(|| {
let prepared: Vec<String> = texts.iter().map(|text| {
let (mut bytes, mut offsets) = (Vec::new(), Vec::new());
normalize_text_into(text, &mut bytes, &mut offsets);
String::from_utf8(bytes).expect("normalizer produces UTF8")
}).collect();
type Match = (usize, usize, usize, usize);
let scan = |basic: bool| -> PyResult<Vec<Match>> {
let mut matches = Vec::new();
for (row, text) in prepared.iter().enumerate() {
let bytes = text.as_bytes();
let mut record = |pattern: usize, start: usize, end: usize| -> PyResult<()> {
if (start == 0 || !is_token_byte(bytes[start - 1]))
&& (end == bytes.len() || !is_token_byte(bytes[end])) {
if matches.len() == 100_000 {
return Err(PyValueError::new_err("matching comparison hit limit exceeded"));
}
matches.push((row, pattern, start, end));
}
Ok(())
};
if basic {
for (pattern, phrase) in self.patterns.iter().enumerate() {
let mut offset = 0;
while let Some(relative) = text[offset..].find(phrase.as_str()) {
let start = offset + relative;
record(pattern, start, start + phrase.len())?;
// Advance one character to retain overlapping matches.
offset = start + text[start..].chars().next().unwrap().len_utf8();
}
}
} else {
for m in self.matcher.find_overlapping_iter(bytes) {
record(m.pattern().as_usize(), m.start(), m.end())?;
}
}
}
Ok(matches)
};
let mut expected = scan(true)?;
let mut actual = scan(false)?;
expected.sort_unstable();
actual.sort_unstable();
if expected != actual {
return Err(PyValueError::new_err("matching comparison outputs differ"));
}
let count = actual.len();
let (mut range_times, mut basic_times) = (Vec::new(), Vec::new());
for round in 0..5 {
for basic in if round % 2 == 0 { [false, true] } else { [true, false] } {
let started = std::time::Instant::now();
let hits = scan(basic)?;
std::hint::black_box(&hits);
let seconds = started.elapsed().as_secs_f64();
if basic { basic_times.push(seconds); } else { range_times.push(seconds); }
}
}
range_times.sort_by(f64::total_cmp);
basic_times.sort_by(f64::total_cmp);
Ok((range_times[2], basic_times[2], count))
})
}
fn scan(
&self,
py: Python<'_>,
texts: Vec<String>,
workers: usize,
max_evidence: usize,
) -> PyResult<Vec<Row>> {
let pool = worker_pool(workers)?;
Ok(py.detach(|| {
pool.install(|| {
texts
.par_iter()
.map_init(
|| (Vec::new(), Vec::new(), vec![[false; 2]; self.rules]),
|(normalized, offsets, found), text| {
normalize_text_into(text, normalized, offsets);
found.fill([false; 2]);
let mut hits = Vec::new();
let mut truncated = false;
for m in self.matcher.find_overlapping_iter(normalized.as_slice()) {
let (start, end) = (m.start(), m.end());
if (start > 0 && is_token_byte(normalized[start - 1]))
|| (end < normalized.len() && is_token_byte(normalized[end]))
{
continue;
}
let (rule, negative) = self.consumers[m.pattern().as_usize()];
found[rule][usize::from(negative)] = true;
if hits.len() < max_evidence {
hits.push((
rule,
negative,
offsets[start],
offsets[end - 1] + 1,
));
} else if max_evidence > 0 {
truncated = true;
}
}
let labels = found
.iter()
.enumerate()
.filter_map(|(i, f)| (f[0] && !f[1]).then_some(i))
.collect();
(labels, hits, truncated)
},
)
.collect()
})
}))
}
}
|