Spaces:
Sleeping
Sleeping
File size: 7,474 Bytes
097eaf0 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 | use napi_derive::napi;
use serde::{Deserialize, Serialize};
use strsim::levenshtein;
use tokio::task;
/// Result of evaluating a single URL across different engines
#[derive(Deserialize, Serialize)]
#[napi(object)]
pub struct EngpickerUrlResult {
pub url: String,
pub cdp_basic_markdown: Option<String>,
pub cdp_basic_success: bool,
pub cdp_stealth_markdown: Option<String>,
pub cdp_stealth_success: bool,
pub tls_basic_markdown: Option<String>,
pub tls_basic_success: bool,
pub tls_stealth_markdown: Option<String>,
pub tls_stealth_success: bool,
}
/// Verdict for a single URL
#[derive(Serialize)]
#[napi(object)]
pub struct EngpickerUrlVerdict {
pub url: String,
pub tls_client_sufficient: bool,
pub cdp_failed: bool,
pub similarity: Option<f64>,
pub reason: String,
}
/// Final verdict enum
#[derive(Serialize)]
#[napi(string_enum)]
pub enum EngpickerFinalVerdict {
/// tlsclient is sufficient for this site
TlsClientOk,
/// Chrome CDP is required for proper rendering
ChromeCdpRequired,
/// Too many CDP failures to determine verdict
Uncertain,
}
/// Final verdict result
#[derive(Serialize)]
#[napi(object)]
pub struct EngpickerVerdict {
pub url_verdicts: Vec<EngpickerUrlVerdict>,
pub tls_client_ok_count: u32,
pub chrome_cdp_required_count: u32,
pub cdp_failed_count: u32,
pub total_urls: u32,
pub verdict: EngpickerFinalVerdict,
}
/// Compute engpicker verdict using Levenshtein distance to compare tlsclient vs chrome-cdp results.
///
/// Chrome-CDP is the gold standard. We compare tlsclient markdown against it to determine
/// if tlsclient is sufficient for scraping this site (i.e., JS rendering not required).
///
/// Arguments:
/// - results: scrape results for each URL
/// - similarity_threshold: minimum similarity (0.0-1.0) for tlsclient to be considered sufficient
/// - success_rate_threshold: minimum ratio of successful comparisons for a definitive verdict
/// - cdp_failure_threshold: maximum ratio of CDP failures before verdict becomes uncertain
#[napi]
pub async fn compute_engpicker_verdict(
results: Vec<EngpickerUrlResult>,
similarity_threshold: f64,
success_rate_threshold: f64,
cdp_failure_threshold: f64,
) -> napi::Result<EngpickerVerdict> {
task::spawn_blocking(move || {
_compute_engpicker_verdict(results, similarity_threshold, success_rate_threshold, cdp_failure_threshold)
})
.await
.map_err(|e| {
napi::Error::new(
napi::Status::GenericFailure,
format!("compute_engpicker_verdict join error: {e}"),
)
})?
}
fn _compute_engpicker_verdict(
results: Vec<EngpickerUrlResult>,
similarity_threshold: f64,
success_rate_threshold: f64,
cdp_failure_threshold: f64,
) -> napi::Result<EngpickerVerdict> {
let url_verdicts: Vec<EngpickerUrlVerdict> = results
.iter()
.map(|result| {
// Get the best chrome-cdp result as gold standard (prefer stealth if both succeeded)
let gold_standard = if result.cdp_stealth_success && result.cdp_stealth_markdown.is_some() {
result.cdp_stealth_markdown.as_ref()
} else if result.cdp_basic_success && result.cdp_basic_markdown.is_some() {
result.cdp_basic_markdown.as_ref()
} else {
None
};
// Get the best tlsclient result (prefer stealth if both succeeded)
let tls_result = if result.tls_stealth_success && result.tls_stealth_markdown.is_some() {
result.tls_stealth_markdown.as_ref()
} else if result.tls_basic_success && result.tls_basic_markdown.is_some() {
result.tls_basic_markdown.as_ref()
} else {
None
};
// If chrome-cdp failed, we can't evaluate this URL
let gold_standard = match gold_standard {
Some(gs) if !gs.is_empty() => gs,
_ => {
return EngpickerUrlVerdict {
url: result.url.clone(),
tls_client_sufficient: false,
cdp_failed: true,
similarity: None,
reason: "chrome-cdp failed".to_string(),
};
}
};
// If tlsclient failed entirely, it's definitely not enough
let tls_result = match tls_result {
Some(tls) if !tls.is_empty() => tls,
_ => {
return EngpickerUrlVerdict {
url: result.url.clone(),
tls_client_sufficient: false,
cdp_failed: false,
similarity: None,
reason: "tlsclient failed".to_string(),
};
}
};
// Calculate Levenshtein distance and normalize to similarity score
let distance = levenshtein(gold_standard, tls_result);
let max_length = gold_standard.len().max(tls_result.len());
let similarity = if max_length > 0 {
1.0 - (distance as f64 / max_length as f64)
} else {
1.0
};
let tls_client_sufficient = similarity >= similarity_threshold;
let reason = if tls_client_sufficient {
format!("{:.1}% similar - tlsclient captures full content", similarity * 100.0)
} else {
format!("{:.1}% similar - JS rendering likely required", similarity * 100.0)
};
EngpickerUrlVerdict {
url: result.url.clone(),
tls_client_sufficient,
cdp_failed: false,
similarity: Some(similarity),
reason,
}
})
.collect();
let total_urls = url_verdicts.len() as u32;
let cdp_failed_count = url_verdicts.iter().filter(|v| v.cdp_failed).count() as u32;
let tls_client_ok_count = url_verdicts.iter().filter(|v| v.tls_client_sufficient).count() as u32;
let chrome_cdp_required_count = url_verdicts.iter().filter(|v| !v.tls_client_sufficient && !v.cdp_failed).count() as u32;
// Determine final verdict
let verdict = if total_urls == 0 {
EngpickerFinalVerdict::Uncertain
} else {
let cdp_failure_rate = cdp_failed_count as f64 / total_urls as f64;
// If too many CDP failures, we can't make a confident verdict
if cdp_failure_rate > cdp_failure_threshold {
EngpickerFinalVerdict::Uncertain
} else {
// Calculate success rate among URLs where we could actually compare
let comparable_urls = total_urls - cdp_failed_count;
if comparable_urls == 0 {
EngpickerFinalVerdict::Uncertain
} else {
let tls_ok_rate = tls_client_ok_count as f64 / comparable_urls as f64;
if tls_ok_rate >= success_rate_threshold {
EngpickerFinalVerdict::TlsClientOk
} else {
EngpickerFinalVerdict::ChromeCdpRequired
}
}
}
};
Ok(EngpickerVerdict {
url_verdicts,
tls_client_ok_count,
chrome_cdp_required_count,
cdp_failed_count,
total_urls,
verdict,
})
}
|