Spaces:
Sleeping
Sleeping
| from bs4 import BeautifulSoup | |
| from app.audit_context import AuditContext | |
| from app.models import CheckResult | |
| _SEMANTIC_TAGS = {"header", "nav", "main", "article", "section", "footer", "aside"} | |
| _AMBIGUOUS_LINK_TEXT = {"", "here", "click here", "more", "learn more", "read more"} | |
| _NON_FIELD_INPUT_TYPES = {"hidden", "submit", "button", "reset", "image"} | |
| _GENERIC_FIELD_NAMES = {"", "field", "input", "value", "data", "form", "formfield", "form_field"} | |
| _VALID_FORM_METHODS = {"get", "post", "dialog"} | |
| _AUTOCOMPLETE_PURPOSE_HINTS = { | |
| "address", | |
| "city", | |
| "company", | |
| "country", | |
| "email", | |
| "family-name", | |
| "first-name", | |
| "given-name", | |
| "last-name", | |
| "name", | |
| "organization", | |
| "phone", | |
| "postal", | |
| "tel", | |
| "username", | |
| "zip", | |
| } | |
| _TEMPLATES = { | |
| "pass": ( | |
| "Page has clear headings, semantic landmarks, useful metadata, and basic " | |
| "agent navigation affordances.", | |
| "No action needed. Your page structure is well-formed for LLM extraction.", | |
| ), | |
| "partial": ( | |
| "Page structure has {issues}. These gaps reduce how accurately LLMs and " | |
| "agents can extract and navigate your content.", | |
| "Fix the flagged issues. Semantic HTML, clear link text, labelled forms, " | |
| "explicit form actions, and useful alt text are fast improvements for " | |
| "AI readability.", | |
| ), | |
| "fail": ( | |
| "Page is missing critical structural elements: {issues}. Agents may " | |
| "struggle to identify the primary content or navigation path.", | |
| "Add a single descriptive H1, organise headings in H1-H2-H3 order, use " | |
| "semantic tags, label form fields, and replace vague link text.", | |
| ), | |
| "fetch_error": ( | |
| "Could not fetch homepage HTML for structural analysis.", | |
| "Ensure the homepage is publicly accessible and returns a 200 response.", | |
| ), | |
| } | |
| def _has_accessible_name(tag) -> bool: | |
| return bool( | |
| tag.get_text(strip=True) | |
| or tag.get("aria-label") | |
| or tag.get("title") | |
| or tag.find("img", alt=True) | |
| ) | |
| def _control_has_accessible_name(tag) -> bool: | |
| if tag.name == "input": | |
| return bool( | |
| tag.get("value", "").strip() | |
| or tag.get("aria-label") | |
| or tag.get("title") | |
| or tag.get("alt") | |
| ) | |
| return _has_accessible_name(tag) | |
| def _form_field_has_label(field, soup: BeautifulSoup) -> bool: | |
| field_id = field.get("id") | |
| if field_id and soup.find("label", attrs={"for": field_id}): | |
| return True | |
| if field.find_parent("label"): | |
| return True | |
| return bool(field.get("aria-label") or field.get("aria-labelledby") or field.get("placeholder")) | |
| def _is_form_field(tag) -> bool: | |
| return tag.name in {"select", "textarea"} or ( | |
| tag.name == "input" and tag.get("type", "text").lower() not in _NON_FIELD_INPUT_TYPES | |
| ) | |
| def _field_name_is_useful(field) -> bool: | |
| name = field.get("name", "").strip().lower() | |
| if name in _GENERIC_FIELD_NAMES: | |
| return False | |
| return bool(name) | |
| def _field_needs_autocomplete(field) -> bool: | |
| if field.name != "input": | |
| return False | |
| field_type = field.get("type", "text").lower() | |
| if field_type in {"email", "tel", "url", "password"}: | |
| return True | |
| haystack = " ".join( | |
| str(value).lower() | |
| for value in ( | |
| field.get("name", ""), | |
| field.get("id", ""), | |
| field.get("aria-label", ""), | |
| field.get("placeholder", ""), | |
| ) | |
| ) | |
| return any(hint in haystack for hint in _AUTOCOMPLETE_PURPOSE_HINTS) | |
| def _field_has_autocomplete(field) -> bool: | |
| value = field.get("autocomplete", "").strip().lower() | |
| return bool(value and value != "off") | |
| def _form_action_is_explicit(action: str | None) -> bool: | |
| return action is not None and bool(action.strip()) | |
| def _form_action_is_non_js(action: str | None) -> bool: | |
| if not _form_action_is_explicit(action): | |
| return False | |
| value = (action or "").strip().lower() | |
| return not value.startswith(("javascript:", "#")) | |
| def _form_method_is_valid(method: str | None) -> bool: | |
| return bool(method and method.strip().lower() in _VALID_FORM_METHODS) | |
| def _form_submit_controls(form) -> list: | |
| controls = [] | |
| for control in form.find_all(["button", "input"]): | |
| control_type = control.get("type", "submit" if control.name == "button" else "text").lower() | |
| if control.name == "button" and control_type in {"", "submit"}: | |
| controls.append(control) | |
| elif control.name == "input" and control_type in {"submit", "image"}: | |
| controls.append(control) | |
| return controls | |
| def _form_has_named_submit_control(form) -> bool: | |
| controls = _form_submit_controls(form) | |
| return bool(controls and any(_control_has_accessible_name(control) for control in controls)) | |
| def _form_has_non_js_fallback(form) -> bool: | |
| return _form_action_is_non_js(form.get("action")) and _form_method_is_valid(form.get("method")) | |
| def _form_affordance_issues(soup: BeautifulSoup) -> list[str]: | |
| forms = soup.find_all("form") | |
| if not forms: | |
| return [] | |
| missing_action = [form for form in forms if not _form_action_is_explicit(form.get("action"))] | |
| js_only_action = [ | |
| form | |
| for form in forms | |
| if _form_action_is_explicit(form.get("action")) | |
| and not _form_action_is_non_js(form.get("action")) | |
| ] | |
| missing_method = [form for form in forms if not form.get("method")] | |
| unsupported_method = [ | |
| form | |
| for form in forms | |
| if form.get("method") and not _form_method_is_valid(form.get("method")) | |
| ] | |
| fields = [ | |
| field | |
| for form in forms | |
| for field in form.find_all(["input", "select", "textarea"]) | |
| if _is_form_field(field) | |
| ] | |
| unnamed_fields = [field for field in fields if not _field_name_is_useful(field)] | |
| autocomplete_fields = [field for field in fields if _field_needs_autocomplete(field)] | |
| missing_autocomplete = [field for field in autocomplete_fields if not _field_has_autocomplete(field)] | |
| unnamed_submit_forms = [form for form in forms if not _form_has_named_submit_control(form)] | |
| fallback_forms = [form for form in forms if _form_has_non_js_fallback(form)] | |
| issues: list[str] = [] | |
| if missing_action: | |
| issues.append(f"{len(missing_action)} form(s) lack explicit action URLs") | |
| if js_only_action: | |
| issues.append(f"{len(js_only_action)} form(s) use JavaScript-only actions") | |
| if missing_method: | |
| issues.append(f"{len(missing_method)} form(s) lack explicit methods") | |
| if unsupported_method: | |
| issues.append(f"{len(unsupported_method)} form(s) use unsupported methods") | |
| if unnamed_fields: | |
| issues.append(f"{len(unnamed_fields)} form field(s) lack useful name attributes") | |
| if missing_autocomplete: | |
| issues.append(f"{len(missing_autocomplete)} form field(s) lack autocomplete hints") | |
| if unnamed_submit_forms: | |
| issues.append(f"{len(unnamed_submit_forms)} form(s) lack named submit controls") | |
| if forms and not fallback_forms: | |
| issues.append("no form exposes an explicit non-JS action/method fallback") | |
| return issues | |
| def _analyse(soup: BeautifulSoup) -> tuple[int, list[str]]: | |
| issues: list[str] = [] | |
| h1_tags = soup.find_all("h1") | |
| if len(h1_tags) == 0: | |
| issues.append("no H1 tag") | |
| elif len(h1_tags) > 1: | |
| issues.append(f"{len(h1_tags)} H1 tags") | |
| h2_tags = soup.find_all("h2") | |
| h3_tags = soup.find_all("h3") | |
| if h3_tags and not h2_tags: | |
| issues.append("H3 headings without any H2") | |
| semantic_present = {tag for tag in _SEMANTIC_TAGS if soup.find(tag)} | |
| if len(semantic_present) < 3: | |
| missing = _SEMANTIC_TAGS - semantic_present | |
| issues.append(f"missing semantic tags: {', '.join(sorted(missing)[:3])}") | |
| title_tag = soup.find("title") | |
| if not title_tag or not title_tag.get_text(strip=True): | |
| issues.append("missing page title") | |
| elif len(title_tag.get_text(strip=True)) > 70: | |
| issues.append("title is unusually long") | |
| meta_desc = soup.find("meta", attrs={"name": "description"}) | |
| if not meta_desc or not meta_desc.get("content", "").strip(): | |
| issues.append("missing meta description") | |
| elif len(meta_desc.get("content", "")) > 180: | |
| issues.append("meta description is unusually long") | |
| links = soup.find_all("a", href=True) | |
| ambiguous_links = [ | |
| link | |
| for link in links | |
| if not _has_accessible_name(link) | |
| or link.get_text(" ", strip=True).lower() in _AMBIGUOUS_LINK_TEXT | |
| ] | |
| if len(links) >= 8 and len(ambiguous_links) / len(links) > 0.25: | |
| issues.append("many links have vague or missing text") | |
| non_crawlable_links = [ | |
| link | |
| for link in links | |
| if link.get("href", "").strip().lower().startswith(("javascript:", "#")) | |
| ] | |
| if len(links) >= 8 and len(non_crawlable_links) / len(links) > 0.25: | |
| issues.append("many navigation links are not crawlable URLs") | |
| fields = [ | |
| field | |
| for field in soup.find_all(["input", "select", "textarea"]) | |
| if _is_form_field(field) | |
| ] | |
| unlabeled_fields = [field for field in fields if not _form_field_has_label(field, soup)] | |
| if unlabeled_fields: | |
| issues.append(f"{len(unlabeled_fields)} form field(s) lack labels") | |
| issues.extend(_form_affordance_issues(soup)) | |
| images = soup.find_all("img") | |
| missing_alt = [image for image in images if image.get("alt") is None] | |
| if len(images) >= 3 and len(missing_alt) / len(images) > 0.5: | |
| issues.append("most images lack alt text") | |
| interactive = soup.find_all(["button", "summary"]) | |
| unnamed_interactive = [tag for tag in interactive if not _control_has_accessible_name(tag)] | |
| if unnamed_interactive: | |
| issues.append(f"{len(unnamed_interactive)} interactive control(s) lack accessible names") | |
| score = max(4 - len(issues), 1) if issues else 4 | |
| return score, issues | |
| async def check_html_structure(context: AuditContext) -> CheckResult: | |
| """Check semantic HTML and static agent navigation affordances.""" | |
| if not context.homepage.ok: | |
| finding, fix = _TEMPLATES["fetch_error"] | |
| return CheckResult( | |
| pillar="scrapability", | |
| check_name="html_structure", | |
| label="Semantic HTML & Agent Navigation", | |
| state="warn", | |
| evidence_level="unknown", | |
| score=0, | |
| max_score=4, | |
| finding=finding, | |
| fix=fix, | |
| effort="medium", | |
| ) | |
| soup = BeautifulSoup(context.homepage.text, "lxml") | |
| score, issues = _analyse(soup) | |
| if not issues: | |
| state = "pass" | |
| finding, fix = _TEMPLATES["pass"] | |
| elif score >= 2: | |
| state = "partial" | |
| finding, fix = _TEMPLATES["partial"] | |
| finding = finding.format(issues="; ".join(issues)) | |
| else: | |
| state = "fail" | |
| finding, fix = _TEMPLATES["fail"] | |
| finding = finding.format(issues="; ".join(issues)) | |
| return CheckResult( | |
| pillar="scrapability", | |
| check_name="html_structure", | |
| label="Semantic HTML & Agent Navigation", | |
| state=state, | |
| score=score, | |
| max_score=4, | |
| finding=finding, | |
| fix=fix, | |
| effort="medium", | |
| ) | |