Buckets:
AnonymousSpottedToad/human_verified_bucket / codebase /python /shangyankeji_super-dev /scripts /add_knowledge_frontmatter.py
| #!/usr/bin/env python3 | |
| """ | |
| 为知识库文件添加 YAML frontmatter 摘要头。 | |
| 让每个知识文件自描述:什么阶段激活、token 预算、关联路径。 | |
| 借鉴 Knowledge Activation 论文的 agent-optimized 摘要头设计。 | |
| frontmatter 格式: | |
| --- | |
| id: owasp-top10-complete | |
| title: OWASP Top 10 完整指南 | |
| domain: security | |
| category: standards | |
| phase_trigger: [backend, quality] | |
| tech_trigger: [python, javascript] | |
| token_budget: 500 | |
| priority: high | |
| summary: "OWASP Top 10 安全风险的完整检查指南" | |
| related: [api-security-complete, supply-chain-security] | |
| --- | |
| 用法: | |
| python scripts/add_knowledge_frontmatter.py # 扫描并添加 | |
| python scripts/add_knowledge_frontmatter.py --dry-run # 仅预览 | |
| python scripts/add_knowledge_frontmatter.py --force # 覆盖已有 frontmatter | |
| 开发:Excellent(11964948@qq.com) | |
| 创建时间:2026-03-28 | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import re | |
| from pathlib import Path | |
| # --------------------------------------------------------------------------- | |
| # 推断规则 | |
| # --------------------------------------------------------------------------- | |
| # domain -> 默认 phase_trigger 映射 | |
| DOMAIN_PHASE_MAP: dict[str, list[str]] = { | |
| "product": ["research", "docs"], | |
| "architecture": ["docs", "spec", "backend"], | |
| "design": ["docs", "frontend"], | |
| "development": ["frontend", "backend", "spec"], | |
| "frontend": ["frontend"], | |
| "backend": ["backend"], | |
| "security": ["backend", "quality", "redteam"], | |
| "testing": ["quality", "qa"], | |
| "operations": ["delivery", "deployment"], | |
| "devops": ["delivery", "deployment"], | |
| "cicd": ["delivery", "deployment"], | |
| "cloud-native": ["backend", "deployment"], | |
| "data": ["backend"], | |
| "data-engineering": ["backend"], | |
| "database": ["backend", "spec"], | |
| "mobile": ["frontend"], | |
| "ai": ["research", "docs", "backend"], | |
| "incident": ["quality", "deployment"], | |
| "edge-iot": ["backend", "deployment"], | |
| "blockchain": ["backend"], | |
| "industries": ["research", "docs"], | |
| "quantum": ["research"], | |
| } | |
| # 文件名关键词 -> tech_trigger 映射 | |
| FILENAME_TECH_MAP: dict[str, list[str]] = { | |
| "react": ["react", "javascript", "typescript"], | |
| "vue": ["vue", "javascript", "typescript"], | |
| "angular": ["angular", "typescript"], | |
| "svelte": ["svelte", "javascript"], | |
| "next": ["react", "nextjs", "typescript"], | |
| "nuxt": ["vue", "nuxt", "typescript"], | |
| "flutter": ["flutter", "dart"], | |
| "react-native": ["react-native", "javascript"], | |
| "python": ["python"], | |
| "django": ["python", "django"], | |
| "fastapi": ["python", "fastapi"], | |
| "flask": ["python", "flask"], | |
| "node": ["node", "javascript", "typescript"], | |
| "express": ["node", "express"], | |
| "nestjs": ["node", "nestjs", "typescript"], | |
| "golang": ["go"], | |
| "go-": ["go"], | |
| "rust": ["rust"], | |
| "java": ["java"], | |
| "spring": ["java", "spring"], | |
| "postgresql": ["postgresql"], | |
| "postgres": ["postgresql"], | |
| "mongodb": ["mongodb"], | |
| "redis": ["redis"], | |
| "mysql": ["mysql"], | |
| "elasticsearch": ["elasticsearch"], | |
| "graphql": ["graphql"], | |
| "rest-api": ["rest"], | |
| "kubernetes": ["kubernetes"], | |
| "docker": ["docker"], | |
| "terraform": ["terraform"], | |
| "airflow": ["python", "airflow"], | |
| "spark": ["spark"], | |
| "playwright": ["playwright", "typescript"], | |
| "jest": ["jest", "javascript"], | |
| "pytest": ["python", "pytest"], | |
| "oauth2": ["oauth2"], | |
| "tailwind": ["tailwind", "css"], | |
| "sass": ["sass", "css"], | |
| } | |
| # category 子目录 -> 标准分类名 | |
| CATEGORY_MAP: dict[str, str] = { | |
| "01-standards": "standards", | |
| "02-playbooks": "playbooks", | |
| "03-checklists": "checklists", | |
| "04-antipatterns": "antipatterns", | |
| "05-cases": "cases", | |
| "06-glossary": "glossary", | |
| "09-maturity": "maturity", | |
| "11-ui-excellence": "standards", | |
| "13-implementation-assets": "playbooks", | |
| "00-governance": "governance", | |
| } | |
| # priority 推断规则:category -> 默认优先级 | |
| CATEGORY_PRIORITY: dict[str, str] = { | |
| "standards": "high", | |
| "checklists": "high", | |
| "antipatterns": "medium", | |
| "playbooks": "medium", | |
| "governance": "medium", | |
| "maturity": "low", | |
| "cases": "low", | |
| "glossary": "low", | |
| } | |
| # --------------------------------------------------------------------------- | |
| # 核心逻辑 | |
| # --------------------------------------------------------------------------- | |
| def has_frontmatter(content: str) -> bool: | |
| """检查文件是否已有 YAML frontmatter""" | |
| return content.startswith("---\n") or content.startswith("---\r\n") | |
| def extract_existing_frontmatter(content: str) -> dict[str, str] | None: | |
| """提取已有的 frontmatter(如果有)""" | |
| if not has_frontmatter(content): | |
| return None | |
| end = content.find("\n---", 3) | |
| if end < 0: | |
| return None | |
| fm_text = content[4:end] | |
| result: dict[str, str] = {} | |
| for line in fm_text.split("\n"): | |
| if ":" in line: | |
| key, _, val = line.partition(":") | |
| result[key.strip()] = val.strip() | |
| return result | |
| def infer_domain(rel_path: Path) -> str: | |
| """从相对路径推断 domain""" | |
| parts = rel_path.parts | |
| if len(parts) >= 1: | |
| return parts[0] | |
| return "unknown" | |
| def infer_category(rel_path: Path) -> str: | |
| """从相对路径推断 category""" | |
| parts = rel_path.parts | |
| if len(parts) >= 2: | |
| second = parts[1] | |
| if re.match(r"^\d{2}-", second): | |
| return CATEGORY_MAP.get(second, second) | |
| # 根目录下的文件,从文件名推断 | |
| filename = rel_path.stem.lower() | |
| if "checklist" in filename: | |
| return "checklists" | |
| if "antipattern" in filename: | |
| return "antipatterns" | |
| if "playbook" in filename or "guide" in filename: | |
| return "playbooks" | |
| if "glossary" in filename: | |
| return "glossary" | |
| if "case" in filename: | |
| return "cases" | |
| if "deep-dive" in filename or "complete" in filename: | |
| return "standards" | |
| return "standards" | |
| def infer_phase_triggers(domain: str, category: str, filename: str) -> list[str]: | |
| """推断文件在哪些 pipeline 阶段被激活""" | |
| phases = list(DOMAIN_PHASE_MAP.get(domain, ["docs"])) | |
| # antipatterns 和 checklists 通常在 quality/redteam 中也激活 | |
| if category in ("antipatterns", "checklists"): | |
| for p in ("quality", "redteam"): | |
| if p not in phases: | |
| phases.append(p) | |
| return phases | |
| def infer_tech_triggers(filename: str, content_head: str) -> list[str]: | |
| """从文件名和内容头部推断关联的技术栈""" | |
| techs: set[str] = set() | |
| filename_lower = filename.lower() | |
| for keyword, tech_list in FILENAME_TECH_MAP.items(): | |
| if keyword in filename_lower: | |
| techs.update(tech_list) | |
| # 如果文件名没匹配到,从内容头部找 | |
| if not techs: | |
| content_lower = content_head.lower() | |
| for keyword, tech_list in FILENAME_TECH_MAP.items(): | |
| # 只在内容前 500 字符中找,避免误匹配 | |
| if keyword in content_lower[:500]: | |
| techs.update(tech_list) | |
| return sorted(techs) | |
| def infer_priority(category: str, file_size: int) -> str: | |
| """推断推送优先级""" | |
| base = CATEGORY_PRIORITY.get(category, "medium") | |
| # 大文件(>50KB)降级,小文件不变 | |
| if file_size > 100_000 and base == "high": | |
| return "medium" | |
| return base | |
| def extract_summary(content: str) -> str: | |
| """从文件的第一段正文提取 summary""" | |
| # 跳过已有 frontmatter | |
| text = content | |
| if has_frontmatter(text): | |
| end = text.find("\n---", 3) | |
| if end > 0: | |
| text = text[end + 4:] | |
| lines = text.split("\n") | |
| paragraphs: list[str] = [] | |
| current: list[str] = [] | |
| for line in lines: | |
| stripped = line.strip() | |
| # 跳过标题行 | |
| if stripped.startswith("#"): | |
| if current: | |
| paragraphs.append(" ".join(current)) | |
| current = [] | |
| continue | |
| # 跳过注释行(# 开头但没有空格的不是标题) | |
| if stripped.startswith("# ") and "开发:" in stripped: | |
| continue | |
| # 跳过空行 | |
| if not stripped: | |
| if current: | |
| paragraphs.append(" ".join(current)) | |
| current = [] | |
| continue | |
| # 跳过代码块 | |
| if stripped.startswith("```"): | |
| if current: | |
| paragraphs.append(" ".join(current)) | |
| current = [] | |
| continue | |
| # 收集正文 | |
| current.append(stripped) | |
| if current: | |
| paragraphs.append(" ".join(current)) | |
| # 取第一个至少 20 字符的段落 | |
| for p in paragraphs: | |
| if len(p) >= 20: | |
| # 截断到 200 字符 | |
| summary = p[:200] | |
| if len(p) > 200: | |
| summary = summary.rsplit(",", 1)[0] if "," in summary[150:] else summary | |
| summary = summary.rsplit(",", 1)[0] if "," in summary[150:] else summary | |
| return summary | |
| return "" | |
| def extract_title(content: str) -> str: | |
| """提取 # 标题""" | |
| text = content | |
| if has_frontmatter(text): | |
| end = text.find("\n---", 3) | |
| if end > 0: | |
| text = text[end + 4:] | |
| for line in text.split("\n"): | |
| stripped = line.strip() | |
| if stripped.startswith("# ") and not stripped.startswith("# 开发:"): | |
| return stripped[2:].strip() | |
| return "" | |
| def infer_related(filename: str, domain: str, all_stems: set[str]) -> list[str]: | |
| """推断关联文件""" | |
| related: list[str] = [] | |
| stem = Path(filename).stem.lower() | |
| # 同 domain 下的其他文件 | |
| # 基于共享关键词 | |
| stem_parts = set(re.split(r"[-_]", stem)) | |
| stem_parts.discard("complete") | |
| stem_parts.discard("deep") | |
| stem_parts.discard("dive") | |
| for other_stem in sorted(all_stems): | |
| if other_stem == stem: | |
| continue | |
| other_parts = set(re.split(r"[-_]", other_stem)) | |
| overlap = stem_parts & other_parts | |
| # 至少有一个有意义的词重叠 | |
| meaningful_overlap = {w for w in overlap if len(w) > 3} | |
| if meaningful_overlap: | |
| related.append(other_stem) | |
| if len(related) >= 5: | |
| break | |
| return related | |
| def estimate_token_budget(file_size: int, priority: str) -> int: | |
| """估算注入 prompt 时的 token 预算""" | |
| if priority == "high": | |
| return min(800, max(300, file_size // 8)) | |
| elif priority == "medium": | |
| return min(500, max(200, file_size // 12)) | |
| else: | |
| return min(300, max(100, file_size // 16)) | |
| def build_frontmatter( | |
| file_id: str, | |
| title: str, | |
| domain: str, | |
| category: str, | |
| phase_trigger: list[str], | |
| tech_trigger: list[str], | |
| token_budget: int, | |
| priority: str, | |
| summary: str, | |
| related: list[str], | |
| ) -> str: | |
| """构建 YAML frontmatter 字符串""" | |
| lines = ["---"] | |
| lines.append(f"id: {file_id}") | |
| lines.append(f"title: \"{title}\"" if title else f"id: {file_id}") | |
| lines.append(f"domain: {domain}") | |
| lines.append(f"category: {category}") | |
| lines.append(f"phase_trigger: [{', '.join(phase_trigger)}]") | |
| if tech_trigger: | |
| lines.append(f"tech_trigger: [{', '.join(tech_trigger)}]") | |
| lines.append(f"token_budget: {token_budget}") | |
| lines.append(f"priority: {priority}") | |
| if summary: | |
| # 转义双引号 | |
| safe_summary = summary.replace('"', '\\"') | |
| lines.append(f'summary: "{safe_summary}"') | |
| if related: | |
| lines.append(f"related: [{', '.join(related)}]") | |
| lines.append("---") | |
| return "\n".join(lines) + "\n" | |
| # --------------------------------------------------------------------------- | |
| # 主流程 | |
| # --------------------------------------------------------------------------- | |
| def process_knowledge_dir( | |
| knowledge_dir: Path, | |
| dry_run: bool = False, | |
| force: bool = False, | |
| ) -> dict[str, int]: | |
| """扫描知识目录,为文件添加 frontmatter | |
| Returns | |
| ------- | |
| dict[str, int] | |
| 统计结果: added, skipped, error | |
| """ | |
| stats = {"added": 0, "skipped": 0, "error": 0, "total": 0} | |
| if not knowledge_dir.is_dir(): | |
| print(f"错误: 知识目录不存在: {knowledge_dir}") | |
| return stats | |
| md_files = sorted(knowledge_dir.rglob("*.md")) | |
| stats["total"] = len(md_files) | |
| # 收集所有文件 stem 用于 related 推断 | |
| all_stems = {f.stem.lower() for f in md_files} | |
| for md_path in md_files: | |
| # 跳过隐藏目录 | |
| if any(part.startswith(".") for part in md_path.parts): | |
| stats["skipped"] += 1 | |
| continue | |
| try: | |
| content = md_path.read_text(encoding="utf-8", errors="replace") | |
| except OSError as e: | |
| print(f" 错误: 无法读取 {md_path}: {e}") | |
| stats["error"] += 1 | |
| continue | |
| # 如果已有 frontmatter 且不 force,跳过 | |
| if has_frontmatter(content) and not force: | |
| stats["skipped"] += 1 | |
| continue | |
| rel_path = md_path.relative_to(knowledge_dir) | |
| domain = infer_domain(rel_path) | |
| category = infer_category(rel_path) | |
| file_id = md_path.stem.lower() | |
| title = extract_title(content) or md_path.stem.replace("-", " ").replace("_", " ").title() | |
| phase_trigger = infer_phase_triggers(domain, category, md_path.name) | |
| tech_trigger = infer_tech_triggers(md_path.name, content[:500]) | |
| priority = infer_priority(category, md_path.stat().st_size) | |
| summary = extract_summary(content) | |
| related = infer_related(md_path.name, domain, all_stems) | |
| token_budget = estimate_token_budget(md_path.stat().st_size, priority) | |
| fm = build_frontmatter( | |
| file_id=file_id, | |
| title=title, | |
| domain=domain, | |
| category=category, | |
| phase_trigger=phase_trigger, | |
| tech_trigger=tech_trigger, | |
| token_budget=token_budget, | |
| priority=priority, | |
| summary=summary, | |
| related=related, | |
| ) | |
| if dry_run: | |
| print(f" [DRY-RUN] {rel_path}") | |
| print(f" domain={domain} category={category} priority={priority}") | |
| print(f" phases={phase_trigger} techs={tech_trigger}") | |
| print(f" summary={summary[:80]}...") | |
| stats["added"] += 1 | |
| continue | |
| # 如果 force 且已有 frontmatter,先移除旧的 | |
| if has_frontmatter(content): | |
| end = content.find("\n---", 3) | |
| if end > 0: | |
| content = content[end + 4:].lstrip("\n") | |
| new_content = fm + "\n" + content | |
| try: | |
| md_path.write_text(new_content, encoding="utf-8") | |
| stats["added"] += 1 | |
| print(f" ✓ {rel_path}") | |
| except OSError as e: | |
| print(f" 错误: 无法写入 {md_path}: {e}") | |
| stats["error"] += 1 | |
| return stats | |
| def main() -> None: | |
| parser = argparse.ArgumentParser( | |
| description="为知识库文件添加 YAML frontmatter 摘要头" | |
| ) | |
| parser.add_argument( | |
| "--knowledge-dir", | |
| type=Path, | |
| default=None, | |
| help="知识库目录路径(默认: 项目根目录/knowledge)", | |
| ) | |
| parser.add_argument( | |
| "--dry-run", | |
| action="store_true", | |
| help="仅预览,不实际写入文件", | |
| ) | |
| parser.add_argument( | |
| "--force", | |
| action="store_true", | |
| help="覆盖已有的 frontmatter", | |
| ) | |
| args = parser.parse_args() | |
| # 查找项目根目录 | |
| knowledge_dir = args.knowledge_dir | |
| if knowledge_dir is None: | |
| # 从脚本位置向上找 | |
| script_dir = Path(__file__).resolve().parent | |
| project_root = script_dir.parent | |
| knowledge_dir = project_root / "knowledge" | |
| print(f"知识库目录: {knowledge_dir}") | |
| print(f"模式: {'预览 (dry-run)' if args.dry_run else '写入'}") | |
| if args.force: | |
| print("强制模式: 将覆盖已有 frontmatter") | |
| print() | |
| stats = process_knowledge_dir(knowledge_dir, dry_run=args.dry_run, force=args.force) | |
| print() | |
| print(f"统计: 总计 {stats['total']} 文件") | |
| print(f" 添加: {stats['added']}") | |
| print(f" 跳过: {stats['skipped']}") | |
| print(f" 错误: {stats['error']}") | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 16.4 kB
- Xet hash:
- 75fff74509fad71bed5e5fae6978a2d3b422a159e21b3291960d67c395a83f70
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.