| |
|
|
| from ultradata_math_parser.utils import * |
| from ultradata_math_parser.config import * |
|
|
|
|
| class TitleParser: |
| def extract_by_meta(self, element: HtmlElement): |
| for xpath in METAS: |
| title = element.xpath(xpath) |
| if title: |
| return "".join(title) |
|
|
| def extract_by_title(self, element: HtmlElement): |
| return "".join(element.xpath("//title//text()")).strip() |
|
|
| def extract_by_hs(self, element: HtmlElement): |
| hs = element.xpath("//h1//text()|//h2//text()|//h3//text()") |
| return hs or [] |
|
|
| def extract_by_h(self, element: HtmlElement): |
| for xpath in ["//h1", "//h2", "//h3"]: |
| children = element.xpath(xpath) |
| if not children: |
| continue |
| child = children[0] |
| texts = child.xpath("./text()") |
| if texts and len(texts): |
| return texts[0].strip() |
|
|
| def process(self, element: HtmlElement): |
| title_extracted_by_meta = self.extract_by_meta(element) |
| if title_extracted_by_meta: |
| return title_extracted_by_meta |
| title_extracted_by_h = self.extract_by_h(element) |
| title_extracted_by_hs = self.extract_by_hs(element) |
| title_extracted_by_title = self.extract_by_title(element) |
| title_extracted_by_hs = sorted( |
| title_extracted_by_hs, |
| key=lambda x: similarity2(x, title_extracted_by_title), |
| reverse=True, |
| ) |
| if title_extracted_by_hs: |
| return lcs_of_2(title_extracted_by_hs[0], title_extracted_by_title) |
|
|
| if title_extracted_by_title: |
| return title_extracted_by_title |
|
|
| return title_extracted_by_h |
|
|
