#!/usr/bin/env python3 """ scripts/seo_audit.py Automated Technical SEO, Metadata, and Crawler Quality Gate for Q-TensorFormer. Validates: 1. All 10 HTML documentation pages in site/ for title, meta description, canonical, OG, Twitter, viewport, charset, and JSON-LD schema validity. 2. XML sitemap structure, URL enumeration, and freshness. 3. robots.txt crawler allowances and sitemap pointer. 4. AI search crawler files (site/llms.txt, site/llms-full.txt, root llms.txt). 5. Internal anchor link integrity across all HTML pages. 6. Hugging Face model card YAML frontmatter and tag integrity. """ import sys import os import re import json import xml.etree.ElementTree as ET from pathlib import Path from html.parser import HTMLParser CANONICAL_BASE = "https://premchan369.github.io/Q-TensorFormer/" EXPECTED_PAGES = [ "index.html", "architecture.html", "methodology.html", "benchmarks.html", "edge-inference.html", "training.html", "documentation.html", "research.html", "limitations.html", "contributing.html", ] class HTMLPageAuditor(HTMLParser): def __init__(self): super().__init__() self.title = None self.in_title = False self.meta_desc = None self.canonical = None self.og_title = None self.og_desc = None self.og_url = None self.viewport = None self.charset = None self.jsonld_scripts = [] self.in_jsonld = False self.current_jsonld = [] self.links = [] def handle_starttag(self, tag, attrs): attrs_dict = dict(attrs) if tag == "title": self.in_title = True elif tag == "meta": if attrs_dict.get("charset"): self.charset = attrs_dict.get("charset") name = attrs_dict.get("name", "").lower() prop = attrs_dict.get("property", "").lower() content = attrs_dict.get("content", "") if name == "description": self.meta_desc = content elif name == "viewport": self.viewport = content elif prop == "og:title": self.og_title = content elif prop == "og:description": self.og_desc = content elif prop == "og:url": self.og_url = content elif tag == "link": rel = attrs_dict.get("rel", "").lower() if rel == "canonical": self.canonical = attrs_dict.get("href") elif tag == "script": script_type = attrs_dict.get("type", "").lower() if script_type == "application/ld+json": self.in_jsonld = True self.current_jsonld = [] elif tag == "a": href = attrs_dict.get("href") if href: self.links.append(href) def handle_endtag(self, tag): if tag == "title": self.in_title = False elif tag == "script" and self.in_jsonld: self.in_jsonld = False self.jsonld_scripts.append("".join(self.current_jsonld).strip()) def handle_data(self, data): if self.in_title: self.title = (self.title or "") + data elif self.in_jsonld: self.current_jsonld.append(data) def run_seo_audit(repo_root: Path) -> bool: site_dir = repo_root / "site" print("=" * 80) print("Q-TensorFormer — Automated Technical SEO & Discoverability Quality Gate") print(f"Auditing Directory: {site_dir}") print("=" * 80) all_passed = True seen_titles = {} seen_descriptions = {} seen_canonicals = set() # 1. Audit HTML Pages print("\n[Phase 1] Auditing HTML Metadata, OpenGraph & JSON-LD Schemas...") for page_name in EXPECTED_PAGES: page_path = site_dir / page_name if not page_path.exists(): print(f" [FAIL] Missing expected documentation page: {page_name}") all_passed = False continue with open(page_path, "r", encoding="utf-8") as f: content = f.read() auditor = HTMLPageAuditor() auditor.feed(content) # Title checks if not auditor.title: print(f" [FAIL] {page_name}: Missing tag") all_passed = False else: title = auditor.title.strip() if title in seen_titles: print(f" [FAIL] {page_name}: Duplicate title '{title}' (first seen in {seen_titles[title]})") all_passed = False else: seen_titles[title] = page_name # Meta description checks if not auditor.meta_desc: print(f" [FAIL] {page_name}: Missing <meta name='description'>") all_passed = False else: desc = auditor.meta_desc.strip() if desc in seen_descriptions: print(f" [FAIL] {page_name}: Duplicate meta description (first seen in {seen_descriptions[desc]})") all_passed = False else: seen_descriptions[desc] = page_name # Canonical checks if not auditor.canonical: print(f" [FAIL] {page_name}: Missing <link rel='canonical'>") all_passed = False else: if auditor.canonical in seen_canonicals: print(f" [FAIL] {page_name}: Duplicate canonical URL '{auditor.canonical}'") all_passed = False seen_canonicals.add(auditor.canonical) expected_canonical = CANONICAL_BASE if page_name == "index.html" else f"{CANONICAL_BASE}{page_name}" if auditor.canonical != expected_canonical: print(f" [FAIL] {page_name}: Canonical mismatch. Expected {expected_canonical}, got {auditor.canonical}") all_passed = False # Viewport and Charset if not auditor.viewport: print(f" [FAIL] {page_name}: Missing responsive viewport meta tag") all_passed = False if not auditor.charset or auditor.charset.lower() != "utf-8": print(f" [FAIL] {page_name}: Charset is missing or not UTF-8") all_passed = False # Open Graph if not (auditor.og_title and auditor.og_desc and auditor.og_url): print(f" [FAIL] {page_name}: Incomplete Open Graph metadata (title/desc/url)") all_passed = False # JSON-LD Schema if not auditor.jsonld_scripts: print(f" [FAIL] {page_name}: Missing JSON-LD structured data") all_passed = False else: for s in auditor.jsonld_scripts: try: data = json.loads(s) if "@context" not in data or "schema.org" not in data["@context"]: print(f" [FAIL] {page_name}: JSON-LD missing valid schema.org @context") all_passed = False if "@type" in data: pass elif "@graph" in data and isinstance(data["@graph"], list): for node in data["@graph"]: if "@type" not in node: print(f" [FAIL] {page_name}: JSON-LD node in @graph missing @type") all_passed = False else: print(f" [FAIL] {page_name}: JSON-LD missing @type or valid @graph") all_passed = False except json.JSONDecodeError as e: print(f" [FAIL] {page_name}: Invalid JSON-LD block: {e}") all_passed = False # Internal Links Verification for link in auditor.links: # strip anchor target = link.split("#")[0] if target and not target.startswith("http") and not target.startswith("mailto"): target_path = (site_dir / target).resolve() if not target_path.exists(): print(f" [FAIL] {page_name}: Broken internal link to '{link}'") all_passed = False print(f" [PASS] {page_name:<20} | Title: {len(auditor.title.strip())}ch | Desc: {len(auditor.meta_desc.strip())}ch | Canonical: OK | Schema: Valid") # 2. Audit XML Sitemap print("\n[Phase 2] Auditing XML Sitemap (site/sitemap.xml)...") sitemap_path = site_dir / "sitemap.xml" if not sitemap_path.exists(): print(" [FAIL] site/sitemap.xml does not exist") all_passed = False else: try: tree = ET.parse(sitemap_path) root = tree.getroot() ns = {"sm": "http://www.sitemaps.org/schemas/sitemap/0.9"} urls = [elem.text.strip() for elem in root.findall(".//sm:loc", ns)] print(f" Found {len(urls)} URLs in sitemap:") for expected_page in EXPECTED_PAGES: expected_url = CANONICAL_BASE if expected_page == "index.html" else f"{CANONICAL_BASE}{expected_page}" if expected_url not in urls: print(f" [FAIL] Missing sitemap entry for {expected_url}") all_passed = False else: print(f" [OK] {expected_url}") except ET.ParseError as e: print(f" [FAIL] XML sitemap parsing failed: {e}") all_passed = False # 3. Audit robots.txt print("\n[Phase 3] Auditing robots.txt...") for r_path, label in [(site_dir / "robots.txt", "site/robots.txt"), (repo_root / "robots.txt", "root robots.txt")]: if not r_path.exists(): print(f" [FAIL] {label} does not exist") all_passed = False else: content = r_path.read_text(encoding="utf-8") if "Sitemap:" not in content: print(f" [FAIL] {label} missing Sitemap directive") all_passed = False else: print(f" [PASS] {label}: Sitemap directive verified") # 4. Audit AI Search Crawler Files (llms.txt) print("\n[Phase 4] Auditing LLM Search Crawler Files (llms.txt & llms-full.txt)...") for f_path, label in [ (site_dir / "llms.txt", "site/llms.txt"), (site_dir / "llms-full.txt", "site/llms-full.txt"), (repo_root / "llms.txt", "root llms.txt"), ]: if not f_path.exists(): print(f" [FAIL] {label} does not exist") all_passed = False else: sz = f_path.stat().st_size if sz < 200: print(f" [FAIL] {label} size is too small ({sz} bytes)") all_passed = False else: print(f" [PASS] {label}: Exists and populated ({sz:,} bytes)") # 5. Audit Hugging Face Model Card YAML frontmatter print("\n[Phase 5] Auditing Hugging Face Model Card & README Metadata...") for md_file in [repo_root / "README.md", repo_root / "MODEL_CARD.md"]: if not md_file.exists(): print(f" [FAIL] {md_file.name} does not exist") all_passed = False continue text = md_file.read_text(encoding="utf-8") if not text.startswith("---"): print(f" [FAIL] {md_file.name} missing YAML frontmatter opening '---'") all_passed = False else: match = re.search(r"^---\n(.*?)\n---", text, re.DOTALL) if not match: print(f" [FAIL] {md_file.name} missing YAML frontmatter closing '---'") all_passed = False else: frontmatter = match.group(1) for req_key in ["tags:", "pipeline_tag:", "license:"]: if req_key not in frontmatter: print(f" [FAIL] {md_file.name} YAML frontmatter missing '{req_key}'") all_passed = False print(f" [PASS] {md_file.name}: Valid YAML frontmatter with required keys") print("\n" + "=" * 80) if all_passed: print("SEO QUALITY GATE RESULT: [PASSED] ALL 10 PAGES & METADATA VERIFIED") print("=" * 80) return True else: print("SEO QUALITY GATE RESULT: [FAILED] REPAIR REPORTED DEFECTS") print("=" * 80) return False if __name__ == "__main__": root = Path(__file__).resolve().parent.parent success = run_seo_audit(root) sys.exit(0 if success else 1)