Blame for scripts/seo_audit.py

#!/usr/bin/env python3
"""
scripts/seo_audit.py

Automated Technical SEO, Metadata, and Crawler Quality Gate for Q-TensorFormer.
Validates:
1. All 10 HTML documentation pages in site/ for title, meta description, canonical, OG, Twitter, viewport, charset, and JSON-LD schema validity.
2. XML sitemap structure, URL enumeration, and freshness.
3. robots.txt crawler allowances and sitemap pointer.
4. AI search crawler files (site/llms.txt, site/llms-full.txt, root llms.txt).
5. Internal anchor link integrity across all HTML pages.
6. Hugging Face model card YAML frontmatter and tag integrity.
"""

import sys
import os
import re
import json
import xml.etree.ElementTree as ET
from pathlib import Path
from html.parser import HTMLParser

CANONICAL_BASE = "https://premchan369.github.io/Q-TensorFormer/"
EXPECTED_PAGES = [
    "index.html",
    "architecture.html",
    "methodology.html",
    "benchmarks.html",
    "edge-inference.html",
    "training.html",
    "documentation.html",
    "research.html",
    "limitations.html",
    "contributing.html",
]

class HTMLPageAuditor(HTMLParser):
    def __init__(self):
        super().__init__()
        self.title = None
        self.in_title = False
        self.meta_desc = None
        self.canonical = None
        self.og_title = None
        self.og_desc = None
        self.og_url = None
        self.viewport = None
        self.charset = None
        self.jsonld_scripts = []
        self.in_jsonld = False
        self.current_jsonld = []
        self.links = []

    def handle_starttag(self, tag, attrs):
        attrs_dict = dict(attrs)
        if tag == "title":
            self.in_title = True
        elif tag == "meta":
            if attrs_dict.get("charset"):
                self.charset = attrs_dict.get("charset")
            name = attrs_dict.get("name", "").lower()
            prop = attrs_dict.get("property", "").lower()
            content = attrs_dict.get("content", "")
            if name == "description":
                self.meta_desc = content
            elif name == "viewport":
                self.viewport = content
            elif prop == "og:title":
                self.og_title = content
            elif prop == "og:description":
                self.og_desc = content
            elif prop == "og:url":
                self.og_url = content
        elif tag == "link":
            rel = attrs_dict.get("rel", "").lower()
            if rel == "canonical":
                self.canonical = attrs_dict.get("href")
        elif tag == "script":
            script_type = attrs_dict.get("type", "").lower()
            if script_type == "application/ld+json":
                self.in_jsonld = True
                self.current_jsonld = []
        elif tag == "a":
            href = attrs_dict.get("href")
            if href:
                self.links.append(href)

    def handle_endtag(self, tag):
        if tag == "title":
            self.in_title = False
        elif tag == "script" and self.in_jsonld:
            self.in_jsonld = False
            self.jsonld_scripts.append("".join(self.current_jsonld).strip())

    def handle_data(self, data):
        if self.in_title:
            self.title = (self.title or "") + data
        elif self.in_jsonld:
            self.current_jsonld.append(data)


def run_seo_audit(repo_root: Path) -> bool:
    site_dir = repo_root / "site"
    print("=" * 80)
    print("Q-TensorFormer — Automated Technical SEO & Discoverability Quality Gate")
    print(f"Auditing Directory: {site_dir}")
    print("=" * 80)

    all_passed = True
    seen_titles = {}
    seen_descriptions = {}
    seen_canonicals = set()

    # 1. Audit HTML Pages
    print("\n[Phase 1] Auditing HTML Metadata, OpenGraph & JSON-LD Schemas...")
    for page_name in EXPECTED_PAGES:
        page_path = site_dir / page_name
        if not page_path.exists():
            print(f"  [FAIL] Missing expected documentation page: {page_name}")
            all_passed = False
            continue

        with open(page_path, "r", encoding="utf-8") as f:
            content = f.read()

        auditor = HTMLPageAuditor()
        auditor.feed(content)

        # Title checks
        if not auditor.title:
            print(f"  [FAIL] {page_name}: Missing <title> tag")
            all_passed = False
        else:
            title = auditor.title.strip()
            if title in seen_titles:
                print(f"  [FAIL] {page_name}: Duplicate title '{title}' (first seen in {seen_titles[title]})")
                all_passed = False
            else:
                seen_titles[title] = page_name

        # Meta description checks
        if not auditor.meta_desc:
            print(f"  [FAIL] {page_name}: Missing <meta name='description'>")
            all_passed = False
        else:
            desc = auditor.meta_desc.strip()
            if desc in seen_descriptions:
                print(f"  [FAIL] {page_name}: Duplicate meta description (first seen in {seen_descriptions[desc]})")
                all_passed = False
            else:
                seen_descriptions[desc] = page_name

        # Canonical checks
        if not auditor.canonical:
            print(f"  [FAIL] {page_name}: Missing <link rel='canonical'>")
            all_passed = False
        else:
            if auditor.canonical in seen_canonicals:
                print(f"  [FAIL] {page_name}: Duplicate canonical URL '{auditor.canonical}'")
                all_passed = False
            seen_canonicals.add(auditor.canonical)
            expected_canonical = CANONICAL_BASE if page_name == "index.html" else f"{CANONICAL_BASE}{page_name}"
            if auditor.canonical != expected_canonical:
                print(f"  [FAIL] {page_name}: Canonical mismatch. Expected {expected_canonical}, got {auditor.canonical}")
                all_passed = False

        # Viewport and Charset
        if not auditor.viewport:
            print(f"  [FAIL] {page_name}: Missing responsive viewport meta tag")
            all_passed = False
        if not auditor.charset or auditor.charset.lower() != "utf-8":
            print(f"  [FAIL] {page_name}: Charset is missing or not UTF-8")
            all_passed = False

        # Open Graph
        if not (auditor.og_title and auditor.og_desc and auditor.og_url):
            print(f"  [FAIL] {page_name}: Incomplete Open Graph metadata (title/desc/url)")
            all_passed = False

        # JSON-LD Schema
        if not auditor.jsonld_scripts:
            print(f"  [FAIL] {page_name}: Missing JSON-LD structured data")
            all_passed = False
        else:
            for s in auditor.jsonld_scripts:
                try:
                    data = json.loads(s)
                    if "@context" not in data or "schema.org" not in data["@context"]:
                        print(f"  [FAIL] {page_name}: JSON-LD missing valid schema.org @context")
                        all_passed = False
                    if "@type" in data:
                        pass
                    elif "@graph" in data and isinstance(data["@graph"], list):
                        for node in data["@graph"]:
                            if "@type" not in node:
                                print(f"  [FAIL] {page_name}: JSON-LD node in @graph missing @type")
                                all_passed = False
                    else:
                        print(f"  [FAIL] {page_name}: JSON-LD missing @type or valid @graph")
                        all_passed = False
                except json.JSONDecodeError as e:
                    print(f"  [FAIL] {page_name}: Invalid JSON-LD block: {e}")
                    all_passed = False

        # Internal Links Verification
        for link in auditor.links:
            # strip anchor
            target = link.split("#")[0]
            if target and not target.startswith("http") and not target.startswith("mailto"):
                target_path = (site_dir / target).resolve()
                if not target_path.exists():
                    print(f"  [FAIL] {page_name}: Broken internal link to '{link}'")
                    all_passed = False

        print(f"  [PASS] {page_name:<20} | Title: {len(auditor.title.strip())}ch | Desc: {len(auditor.meta_desc.strip())}ch | Canonical: OK | Schema: Valid")

    # 2. Audit XML Sitemap
    print("\n[Phase 2] Auditing XML Sitemap (site/sitemap.xml)...")
    sitemap_path = site_dir / "sitemap.xml"
    if not sitemap_path.exists():
        print("  [FAIL] site/sitemap.xml does not exist")
        all_passed = False
    else:
        try:
            tree = ET.parse(sitemap_path)
            root = tree.getroot()
            ns = {"sm": "http://www.sitemaps.org/schemas/sitemap/0.9"}
            urls = [elem.text.strip() for elem in root.findall(".//sm:loc", ns)]
            print(f"  Found {len(urls)} URLs in sitemap:")
            for expected_page in EXPECTED_PAGES:
                expected_url = CANONICAL_BASE if expected_page == "index.html" else f"{CANONICAL_BASE}{expected_page}"
                if expected_url not in urls:
                    print(f"    [FAIL] Missing sitemap entry for {expected_url}")
                    all_passed = False
                else:
                    print(f"    [OK] {expected_url}")
        except ET.ParseError as e:
            print(f"  [FAIL] XML sitemap parsing failed: {e}")
            all_passed = False

    # 3. Audit robots.txt
    print("\n[Phase 3] Auditing robots.txt...")
    for r_path, label in [(site_dir / "robots.txt", "site/robots.txt"), (repo_root / "robots.txt", "root robots.txt")]:
        if not r_path.exists():
            print(f"  [FAIL] {label} does not exist")
            all_passed = False
        else:
            content = r_path.read_text(encoding="utf-8")
            if "Sitemap:" not in content:
                print(f"  [FAIL] {label} missing Sitemap directive")
                all_passed = False
            else:
                print(f"  [PASS] {label}: Sitemap directive verified")

    # 4. Audit AI Search Crawler Files (llms.txt)
    print("\n[Phase 4] Auditing LLM Search Crawler Files (llms.txt & llms-full.txt)...")
    for f_path, label in [
        (site_dir / "llms.txt", "site/llms.txt"),
        (site_dir / "llms-full.txt", "site/llms-full.txt"),
        (repo_root / "llms.txt", "root llms.txt"),
    ]:
        if not f_path.exists():
            print(f"  [FAIL] {label} does not exist")
            all_passed = False
        else:
            sz = f_path.stat().st_size
            if sz < 200:
                print(f"  [FAIL] {label} size is too small ({sz} bytes)")
                all_passed = False
            else:
                print(f"  [PASS] {label}: Exists and populated ({sz:,} bytes)")

    # 5. Audit Hugging Face Model Card YAML frontmatter
    print("\n[Phase 5] Auditing Hugging Face Model Card & README Metadata...")
    for md_file in [repo_root / "README.md", repo_root / "MODEL_CARD.md"]:
        if not md_file.exists():
            print(f"  [FAIL] {md_file.name} does not exist")
            all_passed = False
            continue
        text = md_file.read_text(encoding="utf-8")
        if not text.startswith("---"):
            print(f"  [FAIL] {md_file.name} missing YAML frontmatter opening '---'")
            all_passed = False
        else:
            match = re.search(r"^---\n(.*?)\n---", text, re.DOTALL)
            if not match:
                print(f"  [FAIL] {md_file.name} missing YAML frontmatter closing '---'")
                all_passed = False
            else:
                frontmatter = match.group(1)
                for req_key in ["tags:", "pipeline_tag:", "license:"]:
                    if req_key not in frontmatter:
                        print(f"  [FAIL] {md_file.name} YAML frontmatter missing '{req_key}'")
                        all_passed = False
                print(f"  [PASS] {md_file.name}: Valid YAML frontmatter with required keys")

    print("\n" + "=" * 80)
    if all_passed:
        print("SEO QUALITY GATE RESULT: [PASSED] ALL 10 PAGES & METADATA VERIFIED")
        print("=" * 80)
        return True
    else:
        print("SEO QUALITY GATE RESULT: [FAILED] REPAIR REPORTED DEFECTS")
        print("=" * 80)
        return False


if __name__ == "__main__":
    root = Path(__file__).resolve().parent.parent
    success = run_seo_audit(root)
    sys.exit(0 if success else 1)