Text Generation
Transformers
Safetensors
PyTorch
ONNX
English
qtensorformer
q-tensorformer
high-throughput-inference
tensor-train
tensor-networks
tensorrt-llm
vllm
continuous-batching
paged-attention
speculative-decoding
zero-svd
adaptive-computation
dynamic-neural-networks
kv-cache
kv-cache-quantization
attention-sinks
lyapunov-stability
control-theory
contextual-bandits
edge-ai
edge-inference
on-device-ai
model-compression
hopper-h100
blackwell-b200
jetson-orin
jetson-thor
grouped-query-attention
rotary-position-embeddings
custom_code
academic-paper
ieee
neurips
vit-ap-university
deepseek
deepseek-r1
reasoning
chain-of-thought
gsm8k
openai-compatible
fastapi
rest-api
sse-streaming
onnx-runtime
embedded-ai
hardware-telemetry
power-efficiency
green-ai
raspberry-pi
smollm2
qwen2.5
llama-3.2
apple-silicon
Premchandyadav369
feat(seo): complete global SEO architecture, documentation site, llms.txt, and automated quality gate
f00c0c6 | #!/usr/bin/env python3 | |
| """ | |
| scripts/seo_audit.py | |
| Automated Technical SEO, Metadata, and Crawler Quality Gate for Q-TensorFormer. | |
| Validates: | |
| 1. All 10 HTML documentation pages in site/ for title, meta description, canonical, OG, Twitter, viewport, charset, and JSON-LD schema validity. | |
| 2. XML sitemap structure, URL enumeration, and freshness. | |
| 3. robots.txt crawler allowances and sitemap pointer. | |
| 4. AI search crawler files (site/llms.txt, site/llms-full.txt, root llms.txt). | |
| 5. Internal anchor link integrity across all HTML pages. | |
| 6. Hugging Face model card YAML frontmatter and tag integrity. | |
| """ | |
| import sys | |
| import os | |
| import re | |
| import json | |
| import xml.etree.ElementTree as ET | |
| from pathlib import Path | |
| from html.parser import HTMLParser | |
| CANONICAL_BASE = "https://premchan369.github.io/Q-TensorFormer/" | |
| EXPECTED_PAGES = [ | |
| "index.html", | |
| "architecture.html", | |
| "methodology.html", | |
| "benchmarks.html", | |
| "edge-inference.html", | |
| "training.html", | |
| "documentation.html", | |
| "research.html", | |
| "limitations.html", | |
| "contributing.html", | |
| ] | |
| class HTMLPageAuditor(HTMLParser): | |
| def __init__(self): | |
| super().__init__() | |
| self.title = None | |
| self.in_title = False | |
| self.meta_desc = None | |
| self.canonical = None | |
| self.og_title = None | |
| self.og_desc = None | |
| self.og_url = None | |
| self.viewport = None | |
| self.charset = None | |
| self.jsonld_scripts = [] | |
| self.in_jsonld = False | |
| self.current_jsonld = [] | |
| self.links = [] | |
| def handle_starttag(self, tag, attrs): | |
| attrs_dict = dict(attrs) | |
| if tag == "title": | |
| self.in_title = True | |
| elif tag == "meta": | |
| if attrs_dict.get("charset"): | |
| self.charset = attrs_dict.get("charset") | |
| name = attrs_dict.get("name", "").lower() | |
| prop = attrs_dict.get("property", "").lower() | |
| content = attrs_dict.get("content", "") | |
| if name == "description": | |
| self.meta_desc = content | |
| elif name == "viewport": | |
| self.viewport = content | |
| elif prop == "og:title": | |
| self.og_title = content | |
| elif prop == "og:description": | |
| self.og_desc = content | |
| elif prop == "og:url": | |
| self.og_url = content | |
| elif tag == "link": | |
| rel = attrs_dict.get("rel", "").lower() | |
| if rel == "canonical": | |
| self.canonical = attrs_dict.get("href") | |
| elif tag == "script": | |
| script_type = attrs_dict.get("type", "").lower() | |
| if script_type == "application/ld+json": | |
| self.in_jsonld = True | |
| self.current_jsonld = [] | |
| elif tag == "a": | |
| href = attrs_dict.get("href") | |
| if href: | |
| self.links.append(href) | |
| def handle_endtag(self, tag): | |
| if tag == "title": | |
| self.in_title = False | |
| elif tag == "script" and self.in_jsonld: | |
| self.in_jsonld = False | |
| self.jsonld_scripts.append("".join(self.current_jsonld).strip()) | |
| def handle_data(self, data): | |
| if self.in_title: | |
| self.title = (self.title or "") + data | |
| elif self.in_jsonld: | |
| self.current_jsonld.append(data) | |
| def run_seo_audit(repo_root: Path) -> bool: | |
| site_dir = repo_root / "site" | |
| print("=" * 80) | |
| print("Q-TensorFormer — Automated Technical SEO & Discoverability Quality Gate") | |
| print(f"Auditing Directory: {site_dir}") | |
| print("=" * 80) | |
| all_passed = True | |
| seen_titles = {} | |
| seen_descriptions = {} | |
| seen_canonicals = set() | |
| # 1. Audit HTML Pages | |
| print("\n[Phase 1] Auditing HTML Metadata, OpenGraph & JSON-LD Schemas...") | |
| for page_name in EXPECTED_PAGES: | |
| page_path = site_dir / page_name | |
| if not page_path.exists(): | |
| print(f" [FAIL] Missing expected documentation page: {page_name}") | |
| all_passed = False | |
| continue | |
| with open(page_path, "r", encoding="utf-8") as f: | |
| content = f.read() | |
| auditor = HTMLPageAuditor() | |
| auditor.feed(content) | |
| # Title checks | |
| if not auditor.title: | |
| print(f" [FAIL] {page_name}: Missing <title> tag") | |
| all_passed = False | |
| else: | |
| title = auditor.title.strip() | |
| if title in seen_titles: | |
| print(f" [FAIL] {page_name}: Duplicate title '{title}' (first seen in {seen_titles[title]})") | |
| all_passed = False | |
| else: | |
| seen_titles[title] = page_name | |
| # Meta description checks | |
| if not auditor.meta_desc: | |
| print(f" [FAIL] {page_name}: Missing <meta name='description'>") | |
| all_passed = False | |
| else: | |
| desc = auditor.meta_desc.strip() | |
| if desc in seen_descriptions: | |
| print(f" [FAIL] {page_name}: Duplicate meta description (first seen in {seen_descriptions[desc]})") | |
| all_passed = False | |
| else: | |
| seen_descriptions[desc] = page_name | |
| # Canonical checks | |
| if not auditor.canonical: | |
| print(f" [FAIL] {page_name}: Missing <link rel='canonical'>") | |
| all_passed = False | |
| else: | |
| if auditor.canonical in seen_canonicals: | |
| print(f" [FAIL] {page_name}: Duplicate canonical URL '{auditor.canonical}'") | |
| all_passed = False | |
| seen_canonicals.add(auditor.canonical) | |
| expected_canonical = CANONICAL_BASE if page_name == "index.html" else f"{CANONICAL_BASE}{page_name}" | |
| if auditor.canonical != expected_canonical: | |
| print(f" [FAIL] {page_name}: Canonical mismatch. Expected {expected_canonical}, got {auditor.canonical}") | |
| all_passed = False | |
| # Viewport and Charset | |
| if not auditor.viewport: | |
| print(f" [FAIL] {page_name}: Missing responsive viewport meta tag") | |
| all_passed = False | |
| if not auditor.charset or auditor.charset.lower() != "utf-8": | |
| print(f" [FAIL] {page_name}: Charset is missing or not UTF-8") | |
| all_passed = False | |
| # Open Graph | |
| if not (auditor.og_title and auditor.og_desc and auditor.og_url): | |
| print(f" [FAIL] {page_name}: Incomplete Open Graph metadata (title/desc/url)") | |
| all_passed = False | |
| # JSON-LD Schema | |
| if not auditor.jsonld_scripts: | |
| print(f" [FAIL] {page_name}: Missing JSON-LD structured data") | |
| all_passed = False | |
| else: | |
| for s in auditor.jsonld_scripts: | |
| try: | |
| data = json.loads(s) | |
| if "@context" not in data or "schema.org" not in data["@context"]: | |
| print(f" [FAIL] {page_name}: JSON-LD missing valid schema.org @context") | |
| all_passed = False | |
| if "@type" in data: | |
| pass | |
| elif "@graph" in data and isinstance(data["@graph"], list): | |
| for node in data["@graph"]: | |
| if "@type" not in node: | |
| print(f" [FAIL] {page_name}: JSON-LD node in @graph missing @type") | |
| all_passed = False | |
| else: | |
| print(f" [FAIL] {page_name}: JSON-LD missing @type or valid @graph") | |
| all_passed = False | |
| except json.JSONDecodeError as e: | |
| print(f" [FAIL] {page_name}: Invalid JSON-LD block: {e}") | |
| all_passed = False | |
| # Internal Links Verification | |
| for link in auditor.links: | |
| # strip anchor | |
| target = link.split("#")[0] | |
| if target and not target.startswith("http") and not target.startswith("mailto"): | |
| target_path = (site_dir / target).resolve() | |
| if not target_path.exists(): | |
| print(f" [FAIL] {page_name}: Broken internal link to '{link}'") | |
| all_passed = False | |
| print(f" [PASS] {page_name:<20} | Title: {len(auditor.title.strip())}ch | Desc: {len(auditor.meta_desc.strip())}ch | Canonical: OK | Schema: Valid") | |
| # 2. Audit XML Sitemap | |
| print("\n[Phase 2] Auditing XML Sitemap (site/sitemap.xml)...") | |
| sitemap_path = site_dir / "sitemap.xml" | |
| if not sitemap_path.exists(): | |
| print(" [FAIL] site/sitemap.xml does not exist") | |
| all_passed = False | |
| else: | |
| try: | |
| tree = ET.parse(sitemap_path) | |
| root = tree.getroot() | |
| ns = {"sm": "http://www.sitemaps.org/schemas/sitemap/0.9"} | |
| urls = [elem.text.strip() for elem in root.findall(".//sm:loc", ns)] | |
| print(f" Found {len(urls)} URLs in sitemap:") | |
| for expected_page in EXPECTED_PAGES: | |
| expected_url = CANONICAL_BASE if expected_page == "index.html" else f"{CANONICAL_BASE}{expected_page}" | |
| if expected_url not in urls: | |
| print(f" [FAIL] Missing sitemap entry for {expected_url}") | |
| all_passed = False | |
| else: | |
| print(f" [OK] {expected_url}") | |
| except ET.ParseError as e: | |
| print(f" [FAIL] XML sitemap parsing failed: {e}") | |
| all_passed = False | |
| # 3. Audit robots.txt | |
| print("\n[Phase 3] Auditing robots.txt...") | |
| for r_path, label in [(site_dir / "robots.txt", "site/robots.txt"), (repo_root / "robots.txt", "root robots.txt")]: | |
| if not r_path.exists(): | |
| print(f" [FAIL] {label} does not exist") | |
| all_passed = False | |
| else: | |
| content = r_path.read_text(encoding="utf-8") | |
| if "Sitemap:" not in content: | |
| print(f" [FAIL] {label} missing Sitemap directive") | |
| all_passed = False | |
| else: | |
| print(f" [PASS] {label}: Sitemap directive verified") | |
| # 4. Audit AI Search Crawler Files (llms.txt) | |
| print("\n[Phase 4] Auditing LLM Search Crawler Files (llms.txt & llms-full.txt)...") | |
| for f_path, label in [ | |
| (site_dir / "llms.txt", "site/llms.txt"), | |
| (site_dir / "llms-full.txt", "site/llms-full.txt"), | |
| (repo_root / "llms.txt", "root llms.txt"), | |
| ]: | |
| if not f_path.exists(): | |
| print(f" [FAIL] {label} does not exist") | |
| all_passed = False | |
| else: | |
| sz = f_path.stat().st_size | |
| if sz < 200: | |
| print(f" [FAIL] {label} size is too small ({sz} bytes)") | |
| all_passed = False | |
| else: | |
| print(f" [PASS] {label}: Exists and populated ({sz:,} bytes)") | |
| # 5. Audit Hugging Face Model Card YAML frontmatter | |
| print("\n[Phase 5] Auditing Hugging Face Model Card & README Metadata...") | |
| for md_file in [repo_root / "README.md", repo_root / "MODEL_CARD.md"]: | |
| if not md_file.exists(): | |
| print(f" [FAIL] {md_file.name} does not exist") | |
| all_passed = False | |
| continue | |
| text = md_file.read_text(encoding="utf-8") | |
| if not text.startswith("---"): | |
| print(f" [FAIL] {md_file.name} missing YAML frontmatter opening '---'") | |
| all_passed = False | |
| else: | |
| match = re.search(r"^---\n(.*?)\n---", text, re.DOTALL) | |
| if not match: | |
| print(f" [FAIL] {md_file.name} missing YAML frontmatter closing '---'") | |
| all_passed = False | |
| else: | |
| frontmatter = match.group(1) | |
| for req_key in ["tags:", "pipeline_tag:", "license:"]: | |
| if req_key not in frontmatter: | |
| print(f" [FAIL] {md_file.name} YAML frontmatter missing '{req_key}'") | |
| all_passed = False | |
| print(f" [PASS] {md_file.name}: Valid YAML frontmatter with required keys") | |
| print("\n" + "=" * 80) | |
| if all_passed: | |
| print("SEO QUALITY GATE RESULT: [PASSED] ALL 10 PAGES & METADATA VERIFIED") | |
| print("=" * 80) | |
| return True | |
| else: | |
| print("SEO QUALITY GATE RESULT: [FAILED] REPAIR REPORTED DEFECTS") | |
| print("=" * 80) | |
| return False | |
| if __name__ == "__main__": | |
| root = Path(__file__).resolve().parent.parent | |
| success = run_seo_audit(root) | |
| sys.exit(0 if success else 1) | |