Buckets:

download
raw
71 kB
""" This script processes all repositories in the config/repositories.yml file. Orchestrates the full knowledge base build pipeline (cloning, direct txtai indexing). """ import os import sys # Allow duplicate OpenMP runtime loading when not explicitly configured. This mirrors # the CLI behavior so builds don't abort due to multiple libomp copies. os.environ["KMP_DUPLICATE_LIB_OK"] = "TRUE" import yaml import subprocess from pathlib import Path import logging import argparse import requests import json import time import threading from typing import Optional, Set, Dict try: from dotenv import load_dotenv except Exception: load_dotenv = None from chunky import ChunkPipeline, ChunkerConfig, Document from nancy_brain.chunking import strip_chunk_suffix from nancy_brain.pdf_ocr import extract_pdf_markdown, get_pdf_ocr_backend_status from nancy_brain.summarization import SummaryGenerator from scripts.script_utils import is_full_commit_sha as _is_full_commit_sha # Optional imports try: from nb4llm import convert_ipynb_to_txt except ImportError: convert_ipynb_to_txt = None # Load .env overrides early so subsequent imports see them if load_dotenv is not None: try: script_root = Path(__file__).resolve().parent.parent env_path = script_root / "config" / ".env" if env_path.exists(): load_dotenv(env_path) except Exception: pass # Ensure OpenMP duplicate setting persists after dotenv loads os.environ["KMP_DUPLICATE_LIB_OK"] = "TRUE" # PDF exclusion & thresholds MIN_PDF_BYTES = int(os.environ.get("MIN_PDF_BYTES", "5000")) # skip tiny/image-y PDFs MIN_PDF_TEXT_CHARS = int(os.environ.get("MIN_PDF_TEXT_CHARS", "500")) # require meaningful text DEFAULT_EXCLUDE_PDF_SUBSTRINGS = [ "logo/", "PointSpreadFunctions/", "PSF_", "PSFs_", "PSF-", "PSF.", "/graphics/", "workflow-", "Glossary.pdf", "column-mapping.pdf", ] ENV_EXCLUDES = [e.strip() for e in os.environ.get("PDF_EXCLUDE_SUBSTRINGS", "").split(",") if e.strip()] EXCLUDE_PDF_SUBSTRINGS = DEFAULT_EXCLUDE_PDF_SUBSTRINGS + ENV_EXCLUDES README_CANDIDATES = ["README.md", "README.rst", "README.txt"] DEFAULT_SUMMARIES_ENABLED = os.environ.get("ENABLE_DOC_SUMMARIES", "false").lower() == "true" SKIP_PDFS = os.environ.get("SKIP_PDF_PROCESSING", "").strip().lower() == "true" TEXT_EXTENSIONS = {".py", ".md", ".txt", ".rst", ".tex", ".yaml", ".yml", ".json"} SKIP_DIR_NAMES = { ".git", ".github", "__pycache__", "node_modules", ".pytest_cache", ".mypy_cache", ".idea", ".venv", "env", "venv", } # Minimum content length (stripped) to bother generating a summary. # Override with NB_MIN_SUMMARY_CHARS environment variable. MIN_SUMMARY_CHARS = int(os.environ.get("NB_MIN_SUMMARY_CHARS", "200")) # File extensions for which summary generation is never useful. # These files are still indexed via their path/doc_id. SUMMARY_SKIP_EXTENSIONS = { # Astronomy data ".fits", ".fit", # Numpy/scipy ".npy", ".npz", # Pickle ".pkl", ".pickle", # Tabular ".dat", ".csv", ".parquet", # HDF5 ".hdf5", ".h5", # MATLAB / NIfTI ".mat", ".nii", # Archives ".gz", ".bz2", ".xz", ".zip", ".tar", # Compiled objects ".so", ".o", ".a", ".dylib", ".dll", # Images ".png", ".jpg", ".jpeg", ".gif", ".svg", ".ico", ".woff", ".woff2", } def env_truthy(name: str) -> bool: return os.environ.get(name, "").strip().lower() in {"1", "true", "yes", "on"} def is_excluded_pdf(path: str) -> bool: p = str(path) return any(token in p for token in EXCLUDE_PDF_SUBSTRINGS) def load_repo_readme(repo_dir: Path) -> Optional[dict]: for candidate in README_CANDIDATES: readme_path = repo_dir / candidate if not readme_path.exists(): continue try: text = readme_path.read_text(encoding="utf-8") except Exception: continue if text.strip(): try: relative = str(readme_path.relative_to(repo_dir)) except Exception: relative = str(readme_path.name) return {"content": text, "path": relative} return None def collect_repo_files(repo_dir: Path) -> tuple[list[Path], list[Path]]: text_files: list[Path] = [] pdf_files: list[Path] = [] for root, dirs, files in os.walk(repo_dir): dirs[:] = [d for d in dirs if d not in SKIP_DIR_NAMES] for filename in files: path = Path(root) / filename suffix = path.suffix.lower() if suffix in TEXT_EXTENSIONS or filename.endswith(".nb.txt"): text_files.append(path) elif suffix == ".pdf" and not SKIP_PDFS: pdf_files.append(path) return text_files, pdf_files def emit_progress(percent: int, stage: str = "", detail: str = ""): """Emit a JSON progress marker line to stdout for UIs to parse. Lines are prefixed with PROGRESS_JSON: to make them easy to detect in logs. """ try: payload = {"percent": int(percent), "stage": stage, "detail": detail} print("PROGRESS_JSON: " + json.dumps(payload), flush=True) except Exception: # Never fail the build due to progress emission pass # PDF download headers PDF_REQUEST_HEADERS = { "User-Agent": "nancy-brain-kb-builder/0.1 (+https://example.local)", "Accept": "application/pdf,application/octet-stream;q=0.9,*/*;q=0.8", } def process_pdf_with_ocr(pdf_path, cache_dir=None, use_cached_ocr_only: bool = False): logger = logging.getLogger(__name__) preferred_backend = "skip" if use_cached_ocr_only else None ocr_result = extract_pdf_markdown(pdf_path, cache_dir=cache_dir, preferred_backend=preferred_backend) content = (ocr_result.markdown or "").strip() if not content: if ocr_result.needs_ocr: logger.info("PDF OCR deferred for %s: cache miss and OCR worker unavailable", pdf_path) elif ocr_result.warning: logger.warning("PDF OCR failed for %s: %s", pdf_path, ocr_result.warning) return None, ocr_result, False if len(content) < MIN_PDF_TEXT_CHARS: logger.debug( "Discarding PDF %s (OCR markdown chars %s < %s)", pdf_path, len(content), MIN_PDF_TEXT_CHARS, ) return None, ocr_result, False return content, ocr_result, True # --- Utility function (unchanged from original) --- def get_file_type_category(doc_id: str) -> str: base_id = strip_chunk_suffix(doc_id) path = Path(base_id) code_extensions = { ".py", ".js", ".ts", ".cpp", ".java", ".go", ".rs", ".c", ".h", ".css", ".scss", ".jsx", ".tsx", } if path.suffix in code_extensions: return "code" if ".nb" in path.suffixes or ".nb.txt" in str(path): return "mixed" config_extensions = {".json", ".yaml", ".yml", ".toml", ".ini"} if path.suffix in config_extensions: return "mixed" mixed_extensions = {".md", ".rst"} if path.suffix in mixed_extensions: return "mixed" return "docs" # --- Updated download function with headers --- def download_pdf_articles( config_path: str, base_path: str = "knowledge_base/raw", dry_run: bool = False, category: str = None, force_update: bool = False, ) -> dict: logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s") logger = logging.getLogger(__name__) with open(config_path, "r") as f: config = yaml.safe_load(f) session = requests.Session() session.headers.update(PDF_REQUEST_HEADERS) session.max_redirects = 15 categories = [category] if category else list(config.keys()) failures = { "failed_downloads": [], "skipped_existing": [], "successful_downloads": [], } for cat in categories: articles = config.get(cat) if not isinstance(articles, list): continue for article in articles: article_name = article["name"] article_url = article["url"] dest_dir = Path(base_path) / cat dest_file = dest_dir / f"{article_name}.pdf" if dest_file.exists() and not force_update: logger.info(f"Article {cat}/{article_name}.pdf already exists, skipping.") failures["skipped_existing"].append(f"{cat}/{article_name}") continue logger.info(f"Downloading {article_name} from {article_url} to {dest_file}...") if dry_run: logger.info(f"[DRY RUN] Would download {article_url} to {dest_file}") continue dest_dir.mkdir(parents=True, exist_ok=True) try: resp = session.get(article_url, timeout=45, allow_redirects=True) resp.raise_for_status() ct = resp.headers.get("Content-Type", "") if "text/html" in ct.lower(): raise RuntimeError(f"Expected PDF got HTML (content-type={ct})") if len(resp.content) < MIN_PDF_BYTES: raise RuntimeError(f"PDF too small ({len(resp.content)} bytes < {MIN_PDF_BYTES})") with open(dest_file, "wb") as f: f.write(resp.content) logger.info(f"Successfully downloaded {article_name}") failures["successful_downloads"].append(f"{cat}/{article_name}") except requests.TooManyRedirects as e: logger.error(f"Failed to download {article_name}: Redirect loop ({e})") failures["failed_downloads"].append(f"{cat}/{article_name}: Redirect loop") except Exception as e: if dest_file.exists(): try: dest_file.unlink() except Exception: pass logger.error(f"Failed to download {article_name}: {e}") failures["failed_downloads"].append(f"{cat}/{article_name}: {str(e)}") return failures # --- Clone repositories (original) --- def clone_repositories( config_path: str, base_path: str = "knowledge_base/raw", dry_run: bool = False, category: str = None, force_update: bool = False, repo_filter: Optional[str] = None, ) -> dict: logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s") logger = logging.getLogger(__name__) with open(config_path, "r") as f: config = yaml.safe_load(f) categories = [category] if category else list(config.keys()) failures = { "failed_clones": [], "failed_updates": [], "skipped_existing": [], "successful_clones": [], "successful_updates": [], } for cat in categories: repos = config.get(cat) if not isinstance(repos, list): continue for repo in repos: repo_name = repo["name"] if repo_filter and repo_name != repo_filter: continue repo_url = repo["url"] ref = repo.get("ref") dest_dir = Path(base_path) / cat / repo_name if dest_dir.exists(): if force_update: logger.info(f"Repository {cat}/{repo_name} exists. Updating...") if dry_run: logger.info(f"[DRY RUN] Would update {cat}/{repo_name}") continue try: if ref: if _is_full_commit_sha(ref): fetch_cmd = ["git", "fetch", "origin", ref] else: fetch_cmd = ["git", "fetch", "--depth", "1", "origin", ref] subprocess.run( fetch_cmd, cwd=str(dest_dir), check=True, capture_output=True, text=True, ) subprocess.run( ["git", "checkout", ref], cwd=str(dest_dir), check=True, capture_output=True, text=True, ) else: subprocess.run( ["git", "fetch", "--all"], cwd=str(dest_dir), check=True, capture_output=True, text=True, ) result = subprocess.run( ["git", "rev-parse", "--abbrev-ref", "HEAD"], cwd=str(dest_dir), capture_output=True, text=True, ) current_branch = result.stdout.strip() subprocess.run( ["git", "pull", "origin", current_branch], cwd=str(dest_dir), check=True, capture_output=True, text=True, ) failures["successful_updates"].append(f"{cat}/{repo_name}") except subprocess.CalledProcessError as e: failures["failed_updates"].append(f"{cat}/{repo_name}: {e.stderr}") else: logger.info(f"Repository {cat}/{repo_name} already exists, skipping.") failures["skipped_existing"].append(f"{cat}/{repo_name}") else: logger.info(f"Cloning {repo_name} from {repo_url} into {dest_dir}...") if dry_run: logger.info(f"[DRY RUN] Would clone {repo_url} into {dest_dir}") continue dest_dir.parent.mkdir(parents=True, exist_ok=True) try: if ref: if _is_full_commit_sha(ref): clone_cmd = ["git", "clone", repo_url, str(dest_dir)] else: clone_cmd = ["git", "clone", "--branch", ref, "--depth", "1", repo_url, str(dest_dir)] else: clone_cmd = ["git", "clone", "--depth", "1", repo_url, str(dest_dir)] subprocess.run(clone_cmd, check=True, capture_output=True, text=True) if ref and _is_full_commit_sha(ref): subprocess.run( ["git", "checkout", ref], cwd=str(dest_dir), check=True, capture_output=True, text=True, ) failures["successful_clones"].append(f"{cat}/{repo_name}") except subprocess.CalledProcessError as e: failures["failed_clones"].append(f"{cat}/{repo_name}: {e.stderr}") return failures # --- Build index --- def build_txtai_index( config_path: str, articles_config_path: str = None, base_path: str = "knowledge_base/raw", embeddings_path: str = "knowledge_base/embeddings", dry_run: bool = False, category: str = None, summary_generator: Optional[SummaryGenerator] = None, repo_filter: Optional[str] = None, use_cached_ocr_only: bool = False, ) -> dict: logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s") logger = logging.getLogger(__name__) try: from txtai.embeddings import Embeddings except ImportError: logger.error("txtai not available. Please install with: pip install txtai") return { "failed_text_files": [], "failed_pdf_files": [], "failed_notebook_conversions": [], "deferred_pdf_files": [], "successful_text_files": 0, "successful_pdf_files": 0, "successful_notebook_conversions": 0, "skipped_repositories": [], "skipped_articles": [], } use_dual_embedding = os.environ.get("USE_DUAL_EMBEDDING", "true").lower() == "true" code_model = os.environ.get("CODE_EMBEDDING_MODEL", "microsoft/codebert-base") logger.info(f"Dual embedding enabled: {use_dual_embedding}") if use_dual_embedding: logger.info(f"Code model: {code_model}") if SKIP_PDFS: logger.info("SKIP_PDF_PROCESSING enabled; PDF documents will be ignored during build.") pdf_ocr_status = get_pdf_ocr_backend_status() if not SKIP_PDFS: if pdf_ocr_status.available: logger.info("✅ PDF OCR backend selected: %s (%s)", pdf_ocr_status.name, pdf_ocr_status.model) else: logger.warning( "PDF OCR unavailable; PDFs will be skipped unless OCR extras are installed: %s", pdf_ocr_status.reason ) try: with open(config_path, "r") as f: config = yaml.safe_load(f) except Exception as e: logger.error(f"Failed to read repository config: {config_path} ({e})") return { "failed_text_files": [], "failed_pdf_files": [], "failed_notebook_conversions": [], "deferred_pdf_files": [], "successful_text_files": 0, "successful_pdf_files": 0, "successful_notebook_conversions": 0, "skipped_repositories": [], "skipped_articles": [], "validation_errors": [f"Failed to read config: {config_path}: {e}"], } # Validate repository config early to give clear feedback try: from nancy_brain.config_validation import validate_repositories_config, validate_articles_config ok, errors = validate_repositories_config(config or {}) if not ok: logger.error("repositories.yml validation failed") return { "failed_text_files": [], "failed_pdf_files": [], "failed_notebook_conversions": [], "deferred_pdf_files": [], "successful_text_files": 0, "successful_pdf_files": 0, "successful_notebook_conversions": 0, "skipped_repositories": [], "skipped_articles": [], "validation_errors": errors, } except Exception: # If validation helpers are unavailable for any reason, continue without failing pass articles_config = {} if articles_config_path and os.path.exists(articles_config_path): with open(articles_config_path, "r") as f: articles_config = yaml.safe_load(f) logger.info(f"Loaded articles configuration from {articles_config_path}") # Validate articles config if validator available try: ok2, errors2 = validate_articles_config(articles_config or {}) if not ok2: logger.error("articles.yml validation failed") return { "failed_text_files": [], "failed_pdf_files": [], "failed_notebook_conversions": [], "deferred_pdf_files": [], "successful_text_files": 0, "successful_pdf_files": 0, "successful_notebook_conversions": 0, "skipped_repositories": [], "skipped_articles": [], "validation_errors": errors2, } except Exception: pass embeddings_dir = Path(embeddings_path) embeddings_dir.mkdir(parents=True, exist_ok=True) text_model = os.environ.get("NB_TEXT_EMBEDDING_MODEL", "sentence-transformers/all-MiniLM-L6-v2") logger.info(f"Loading general embedding model: {text_model}") model_start = time.time() try: general_embeddings = Embeddings( { "path": text_model, "content": True, "backend": "faiss", } ) except Exception as exc: logger.error(f"Failed to load general embedding model '{text_model}': {exc}") logger.error( "If this is the first run, ensure the model is reachable or set NB_TEXT_EMBEDDING_MODEL" " to a smaller/local SentenceTransformer." ) raise logger.info(f"General embedding model ready in {time.time() - model_start:.2f}s") code_embeddings = None if use_dual_embedding: code_model_override = os.environ.get("NB_CODE_EMBEDDING_MODEL", code_model) logger.info(f"Loading code embedding model: {code_model_override}") code_start = time.time() try: code_embeddings = Embeddings({"path": code_model_override, "content": True, "backend": "faiss"}) except Exception as exc: logger.error(f"Failed to load code embedding model '{code_model_override}': {exc}") logger.error( "Disable dual embeddings with USE_DUAL_EMBEDDING=false or set NB_CODE_EMBEDDING_MODEL" " to a lighter model." ) raise logger.info(f"Code embedding model ready in {time.time() - code_start:.2f}s") try: with open("config/index_weights.yaml", "r") as f: ext_weights = yaml.safe_load(f) except Exception: ext_weights = {} categories = [category] if category else list(config.keys()) # Count total repositories for progress tracking total_repos = 0 for cat in categories: repos_list = config.get(cat) if isinstance(repos_list, list): if repo_filter: total_repos += sum(1 for repo in repos_list if repo.get("name") == repo_filter) else: total_repos += len(repos_list) repos_processed = 0 documents = [] pipeline = ChunkPipeline() pdf_ocr_cache_dir = Path(base_path).parent / "cache" / "pdf_ocr" chunk_config = ChunkerConfig( max_chars=int(os.environ.get("CHUNKY_MAX_CHARS", "1000")), lines_per_chunk=int(os.environ.get("CHUNKY_LINES_PER_CHUNK", "40")), line_overlap=int(os.environ.get("CHUNKY_LINE_OVERLAP", "5")), ) pdf_status = {} # doc_id -> {status, size, chars, method} failures = { "failed_text_files": [], "failed_pdf_files": [], "failed_notebook_conversions": [], "deferred_pdf_files": [], "successful_text_files": 0, "successful_pdf_files": 0, "successful_notebook_conversions": 0, "skipped_repositories": [], "skipped_articles": [], "skipped_low_value": [], "fatal_errors": [], } text_candidate_count = 0 pdf_candidate_count = 0 summary_documents = [] summary_enabled = False if summary_generator is not None: summary_enabled = bool(getattr(summary_generator, "enabled", True)) summary_stats = { "configured": summary_generator is not None, "enabled": summary_enabled, "requests": 0, "responses": 0, "cached_hits": 0, "failures": 0, "documents_added": 0, } auto_model_weights: Dict[str, float] = {} summarized_docs: Set[str] = set() repo_readme_cache: Dict[Path, Optional[dict]] = {} def should_skip_summary(doc_id: str, content: str, file_path: Optional[Path]) -> bool: stripped = content.strip() if len(stripped) < MIN_SUMMARY_CHARS: logger.debug( "Skipping summary for %s: content too short (%d chars < %d)", doc_id, len(stripped), MIN_SUMMARY_CHARS, ) return True if file_path is not None and file_path.suffix.lower() in SUMMARY_SKIP_EXTENSIONS: logger.debug( "Skipping summary for %s: extension %s in SUMMARY_SKIP_EXTENSIONS", doc_id, file_path.suffix.lower(), ) return True return False for cat in categories: repos = config.get(cat) if not isinstance(repos, list): continue for repo in repos: repo_name = repo["name"] if repo_filter and repo_name != repo_filter: continue # Emit repo progress (50-85% range) repos_processed += 1 pct = 50 + int(((repos_processed - 0.5) / max(1, total_repos)) * 35) emit_progress(pct, stage="indexing", detail=f"Indexing {repo_name} ({repos_processed}/{total_repos})") repo_dir = Path(base_path) / cat / repo_name if not repo_dir.exists(): failures["skipped_repositories"].append(f"{cat}/{repo_name}") continue repo_readme_content = None repo_readme_path = None if summary_generator is not None: if repo_dir not in repo_readme_cache: repo_readme_cache[repo_dir] = load_repo_readme(repo_dir) repo_readme_info = repo_readme_cache.get(repo_dir) or {} repo_readme_content = repo_readme_info.get("content") repo_readme_path = repo_readme_info.get("path") if convert_ipynb_to_txt is not None: for ipynb_file in repo_dir.rglob("*.ipynb"): nb_txt_file = ipynb_file.with_suffix(".nb.txt") if not nb_txt_file.exists() or ipynb_file.stat().st_mtime > nb_txt_file.stat().st_mtime: try: convert_ipynb_to_txt(str(ipynb_file), str(nb_txt_file)) failures["successful_notebook_conversions"] += 1 except Exception as e: failures["failed_notebook_conversions"].append(f"{ipynb_file}: {str(e)}") text_files, pdf_files = collect_repo_files(repo_dir) if SKIP_PDFS: pdf_files = [] else: filtered_pdf_files = [] for pf in pdf_files: try: if is_excluded_pdf(pf): pdf_status[str(pf)] = { "status": "excluded", "reason": "pattern", "size": pf.stat().st_size if pf.exists() else 0, } continue size = pf.stat().st_size if size < MIN_PDF_BYTES: pdf_status[str(pf)] = { "status": "skipped", "reason": f"small({size})", "size": size, } continue filtered_pdf_files.append(pf) except Exception: continue pdf_files = filtered_pdf_files text_files = [f for f in text_files if "docs/build" not in str(f) and not str(f).endswith(".rst.txt")] if ( os.environ.get("NB_SCAN_LOG", "false").lower() == "true" or os.environ.get("NB_PER_FILE_LOG", "false").lower() == "true" ): logger.info( f"Repo {repo_name}: {len(text_files)} candidate text files, {len(pdf_files)} candidate PDFs" ) # Use text extraction helpers for .rst and .tex files when available try: from scripts.text_extract import extract_text_from_rst, extract_text_from_tex except Exception: extract_text_from_rst = None extract_text_from_tex = None for file_path in text_files: try: text_candidate_count += 1 logger.info(f"Reading file {file_path.relative_to(repo_dir)}") with open(file_path, "r", encoding="utf-8") as f: content = f.read() if not content.strip(): continue suffix = file_path.suffix.lower() if suffix == ".rst" and extract_text_from_rst: try: content = extract_text_from_rst(content) except Exception: pass if suffix == ".tex" and extract_text_from_tex: try: content = extract_text_from_tex(content) except Exception: pass if not content or not content.strip(): continue relative_path = file_path.relative_to(repo_dir) doc_id = f"{cat}/{repo_name}/{relative_path}" logger.info(f"Chunking file {doc_id}") if summary_generator is not None and summary_stats["enabled"] and doc_id not in summarized_docs: if not should_skip_summary(doc_id, content, file_path): summary_stats["requests"] += 1 summary_payload = None failed_due_to_exception = False try: summary_payload = summary_generator.summarize( doc_id=doc_id, content=content, repo_name=repo_name, repo_readme=repo_readme_content, repo_readme_path=repo_readme_path, repo_description=repo.get("description"), metadata={"category": cat, "source": "repository_file"}, ) except Exception as exc: failed_due_to_exception = True summary_stats["failures"] += 1 logger.warning("Summary generation failed for %s: %s", doc_id, exc) if summary_payload is not None: summary_stats["responses"] += 1 if summary_payload.cached: summary_stats["cached_hits"] += 1 summary_meta = { "source_document": doc_id, "category": cat, "repository": repo_name, "doc_type": "summary", "model": summary_payload.model, "cached": summary_payload.cached, } if summary_payload.repo_readme_path: summary_meta["repo_readme_path"] = summary_payload.repo_readme_path summary_documents.append( ( f"summaries/{doc_id}", summary_payload.summary, json.dumps(summary_meta, ensure_ascii=False), ) ) auto_model_weights[doc_id] = summary_payload.weight summarized_docs.add(doc_id) elif not failed_due_to_exception: summary_stats["failures"] += 1 document = Document(path=file_path, content=content, metadata={"doc_id": doc_id}) chunks = pipeline.chunk_documents([document], config=chunk_config) if not chunks: logger.debug(f"Skipping low-value document: {doc_id}") failures["skipped_low_value"].append(doc_id) continue for chunk in chunks: meta = dict(chunk.metadata) meta.setdefault("source_document", doc_id) meta.setdefault("extension", suffix) meta.setdefault("relative_path", str(relative_path)) meta.setdefault("repository", repo_name) documents.append( ( chunk.chunk_id, chunk.text, json.dumps(meta, ensure_ascii=False), ) ) failures["successful_text_files"] += 1 except Exception as e: failures["failed_text_files"].append(f"{file_path}: {str(e)}") for pdf_path in pdf_files: try: pdf_candidate_count += 1 content, ocr_result, success = process_pdf_with_ocr( pdf_path, cache_dir=pdf_ocr_cache_dir, use_cached_ocr_only=use_cached_ocr_only, ) size = pdf_path.stat().st_size if pdf_path.exists() else 0 if success and content: relative_path = pdf_path.relative_to(repo_dir) doc_id = f"{cat}/{repo_name}/{relative_path}" metadata = ( f"# Repository PDF\n\n" f"- Repository: {repo_name}\n" f"- Source: {repo['url']}\n" f"- Path: `{relative_path}`\n" f"- OCR backend: {ocr_result.backend}\n" f"- OCR cache: {'hit' if ocr_result.cached else 'miss'}\n\n" ) full_content = metadata + content if summary_generator is not None and summary_stats["enabled"] and doc_id not in summarized_docs: if not should_skip_summary(doc_id, full_content, pdf_path): summary_stats["requests"] += 1 summary_payload = None failed_due_to_exception = False try: summary_payload = summary_generator.summarize( doc_id=doc_id, content=full_content, repo_name=repo_name, repo_readme=repo_readme_content, repo_readme_path=repo_readme_path, metadata={"category": cat, "source": "repository_pdf"}, ) except Exception as exc: failed_due_to_exception = True summary_stats["failures"] += 1 logger.warning("Summary generation failed for %s: %s", doc_id, exc) if summary_payload is not None: summary_stats["responses"] += 1 if summary_payload.cached: summary_stats["cached_hits"] += 1 summary_meta = { "source_document": doc_id, "category": cat, "repository": repo_name, "doc_type": "summary", "model": summary_payload.model, "cached": summary_payload.cached, } if summary_payload.repo_readme_path: summary_meta["repo_readme_path"] = summary_payload.repo_readme_path summary_documents.append( ( f"summaries/{doc_id}", summary_payload.summary, json.dumps(summary_meta, ensure_ascii=False), ) ) auto_model_weights[doc_id] = summary_payload.weight summarized_docs.add(doc_id) elif not failed_due_to_exception: summary_stats["failures"] += 1 markdown_path = Path(f"{pdf_path}.ocr.md") document = Document(path=markdown_path, content=full_content, metadata={"doc_id": doc_id}) chunks = pipeline.chunk_documents([document], config=chunk_config) if not chunks: logger.debug(f"Skipping low-value PDF: {doc_id}") failures["skipped_low_value"].append(doc_id) continue for chunk in chunks: meta = dict(chunk.metadata) meta.setdefault("source_document", doc_id) meta.setdefault("relative_path", str(relative_path)) meta.setdefault("repository", repo_name) meta.setdefault("pdf_size_bytes", size) meta.setdefault("source_url", repo.get("url")) meta.setdefault("ocr_backend", ocr_result.backend) meta.setdefault("ocr_cached", ocr_result.cached) meta.setdefault("ocr_cache_key", ocr_result.cache_key) documents.append( ( chunk.chunk_id, chunk.text, json.dumps(meta, ensure_ascii=False), ) ) failures["successful_pdf_files"] += 1 pdf_status[doc_id] = { "status": "indexed", "size": size, "chars": len(content), "chunk_count": len(chunks), "backend": ocr_result.backend, "cached": ocr_result.cached, "cache_key": ocr_result.cache_key, "page_count": ocr_result.page_count, } elif ocr_result.needs_ocr and use_cached_ocr_only: failures["deferred_pdf_files"].append(str(pdf_path)) pdf_status[str(pdf_path)] = { "status": "needs_ocr", "size": size, "backend": ocr_result.backend, "reason": ocr_result.warning, } else: reason = ocr_result.warning or "No meaningful markdown extracted" failures["failed_pdf_files"].append(f"{pdf_path}: {reason}") pdf_status[str(pdf_path)] = { "status": "failed_extract", "size": size, "backend": ocr_result.backend, "reason": reason, } except Exception as e: failures["failed_pdf_files"].append(f"{pdf_path}: {str(e)}") pdf_status[str(pdf_path)] = { "status": "error", "error": str(e)[:120], } article_categories = [category] if category else list(articles_config.keys()) for cat in article_categories: articles = articles_config.get(cat) if not isinstance(articles, list): continue for article in articles: article_name = article["name"] pdf_file = Path(base_path) / cat / f"{article_name}.pdf" if not pdf_file.exists(): failures["skipped_articles"].append(f"{cat}/{article_name}") continue try: pdf_candidate_count += 1 content, ocr_result, success = process_pdf_with_ocr( pdf_file, cache_dir=pdf_ocr_cache_dir, use_cached_ocr_only=use_cached_ocr_only, ) if success and content: doc_id = f"journal_articles/{cat}/{article_name}" metadata = ( f"# Journal Article\n\n" f"- Title: {article['description']}\n" f"- Source: {article.get('url', 'Unknown')}\n" f"- OCR backend: {ocr_result.backend}\n" f"- OCR cache: {'hit' if ocr_result.cached else 'miss'}\n\n" ) full_content = metadata + content if summary_generator is not None and summary_stats["enabled"] and doc_id not in summarized_docs: if not should_skip_summary(doc_id, full_content, pdf_file): summary_stats["requests"] += 1 summary_payload = None failed_due_to_exception = False try: summary_payload = summary_generator.summarize( doc_id=doc_id, content=full_content, repo_name=cat, repo_readme=None, repo_readme_path=None, metadata={"category": cat, "source": "journal_pdf", "article": article_name}, ) except Exception as exc: failed_due_to_exception = True summary_stats["failures"] += 1 logger.warning("Summary generation failed for %s: %s", doc_id, exc) if summary_payload is not None: summary_stats["responses"] += 1 if summary_payload.cached: summary_stats["cached_hits"] += 1 summary_meta = { "source_document": doc_id, "category": cat, "doc_type": "summary", "model": summary_payload.model, "cached": summary_payload.cached, } if summary_payload.repo_readme_path: summary_meta["repo_readme_path"] = summary_payload.repo_readme_path summary_documents.append( ( f"summaries/{doc_id}", summary_payload.summary, json.dumps(summary_meta, ensure_ascii=False), ) ) auto_model_weights[doc_id] = summary_payload.weight summarized_docs.add(doc_id) elif not failed_due_to_exception: summary_stats["failures"] += 1 markdown_path = Path(f"{pdf_file}.ocr.md") document = Document(path=markdown_path, content=full_content, metadata={"doc_id": doc_id}) chunks = pipeline.chunk_documents([document], config=chunk_config) if not chunks: logger.debug(f"Skipping low-value journal PDF: {doc_id}") failures["skipped_low_value"].append(doc_id) continue size_bytes = pdf_file.stat().st_size if pdf_file.exists() else 0 for chunk in chunks: meta = dict(chunk.metadata) meta.setdefault("source_document", doc_id) meta.setdefault("article_name", article_name) meta.setdefault("category", meta.get("category", "docs")) meta.setdefault("pdf_size_bytes", size_bytes) meta.setdefault("source_url", article.get("url")) meta.setdefault("ocr_backend", ocr_result.backend) meta.setdefault("ocr_cached", ocr_result.cached) meta.setdefault("ocr_cache_key", ocr_result.cache_key) documents.append( ( chunk.chunk_id, chunk.text, json.dumps(meta, ensure_ascii=False), ) ) failures["successful_pdf_files"] += 1 pdf_status[doc_id] = { "status": "indexed", "size": size_bytes, "chars": len(content), "chunk_count": len(chunks), "backend": ocr_result.backend, "cached": ocr_result.cached, "cache_key": ocr_result.cache_key, "page_count": ocr_result.page_count, } elif ocr_result.needs_ocr and use_cached_ocr_only: failures["deferred_pdf_files"].append(str(pdf_file)) pdf_status[str(pdf_file)] = { "status": "needs_ocr", "backend": ocr_result.backend, "reason": ocr_result.warning, } else: reason = ocr_result.warning or "No meaningful markdown extracted" failures["failed_pdf_files"].append(f"{pdf_file}: {reason}") pdf_status[str(pdf_file)] = { "status": "failed_extract", "backend": ocr_result.backend, "reason": reason, } except Exception as e: failures["failed_pdf_files"].append(f"{pdf_file}: {str(e)}") pdf_status[str(pdf_file)] = { "status": "error", "error": str(e)[:120], } summary_stats["documents_added"] = len(summary_documents) failures["summary_stats"] = summary_stats if summary_documents: documents.extend(summary_documents) total_indexed_documents = len(documents) failures["indexed_document_count"] = total_indexed_documents failures["text_file_candidates"] = text_candidate_count failures["pdf_file_candidates"] = pdf_candidate_count if not documents: if not dry_run: if text_candidate_count or pdf_candidate_count: deferred_pdfs = failures.get("deferred_pdf_files") or [] if ( use_cached_ocr_only and deferred_pdfs and not failures.get("failed_text_files") and not failures.get("failed_pdf_files") ): logger.warning( "No documents were indexed because %s PDFs are deferred awaiting OCR cache or worker availability.", len(deferred_pdfs), ) else: failures["fatal_errors"].append( "No documents were indexed even though source files were discovered. " "Check for upstream processing failures (e.g., summarization errors) in the logs." ) else: failures["fatal_errors"].append( "No documents were indexed because no eligible source files were found in the configured sources." ) else: logger.info("Dry run: no documents would be indexed.") elif not dry_run: logger.info(f"Indexing {len(documents)} documents...") logger.info("Building general embeddings index… (first run may download embedding model weights)") start = time.time() general_embeddings.index(documents) logger.info(f"General index built in {time.time() - start:.2f}s") general_embeddings.save(str(embeddings_dir / "index")) try: index_dir = embeddings_dir / "index" cfg_json = index_dir / "config.json" cfg = index_dir / "config" if cfg_json.exists() and not cfg.exists(): cfg.write_text(cfg_json.read_text(encoding="utf-8"), encoding="utf-8") except Exception as exc: logger.warning(f"Failed to write txtai config shim for index: {exc}") if use_dual_embedding and code_embeddings: logger.info("Building code embeddings index...") cstart = time.time() code_embeddings.index(documents) logger.info(f"Code index built in {time.time() - cstart:.2f}s") code_embeddings.save(str(embeddings_dir / "code_index")) try: code_index_dir = embeddings_dir / "code_index" cfg_json = code_index_dir / "config.json" cfg = code_index_dir / "config" if cfg_json.exists() and not cfg.exists(): cfg.write_text(cfg_json.read_text(encoding="utf-8"), encoding="utf-8") except Exception as exc: logger.warning(f"Failed to write txtai config shim for code_index: {exc}") results = general_embeddings.search("function", 3) logger.info("Test search results (general model):") for result in results: logger.info(f" - {result['id']}: {result['text'][:100]}...") if use_dual_embedding and code_embeddings: code_results = code_embeddings.search("function", 3) logger.info("Test search results (code model):") for result in code_results: logger.info(f" - {result['id']}: {result['text'][:100]}...") elif not documents: logger.warning("No documents found to index") # Write manifest if not dry_run: try: with open(embeddings_dir / "pdf_manifest.json", "w", encoding="utf-8") as mf: json.dump(pdf_status, mf, indent=2) logger.info(f"Wrote PDF manifest to {embeddings_dir / 'pdf_manifest.json'}") except Exception as e: logger.warning(f"Failed to write pdf_manifest.json: {e}") if auto_model_weights: try: with open(embeddings_dir / "auto_model_weights.json", "w", encoding="utf-8") as wf: json.dump(auto_model_weights, wf, indent=2) logger.info( f"Wrote suggested model weights for {len(auto_model_weights)} docs to" f" {embeddings_dir / 'auto_model_weights.json'}" ) except Exception as exc: logger.warning(f"Failed to write auto_model_weights.json: {exc}") if auto_model_weights: failures["auto_model_weights"] = auto_model_weights return failures # --- Cleanup & summary functions (original) --- def cleanup_pdf_articles( articles_config_path: str, base_path: str = "knowledge_base/raw", category: str = None, ) -> None: """Clean up downloaded PDF articles after embeddings are built""" logger = logging.getLogger(__name__) with open(articles_config_path, "r") as f: config = yaml.safe_load(f) categories = [category] if category else list(config.keys()) for cat in categories: articles = config.get(cat) if not isinstance(articles, list): continue for article in articles: article_name = article["name"] pdf_file = Path(base_path) / cat / f"{article_name}.pdf" if pdf_file.exists(): try: pdf_file.unlink() logger.info(f"Cleaned up PDF {cat}/{article_name}.pdf") except Exception as e: logger.warning(f"Failed to clean up PDF {cat}/{article_name}.pdf: {e}") # Clean up empty category directories base_path_obj = Path(base_path) if base_path_obj.exists(): for cat_dir in base_path_obj.iterdir(): if cat_dir.is_dir() and not any(cat_dir.iterdir()): try: cat_dir.rmdir() logger.info(f"Cleaned up empty category directory: {cat_dir}") except Exception as e: logger.warning(f"Failed to clean up empty category directory {cat_dir}: {e}") def cleanup_raw_repositories(config_path: str, base_path: str = "knowledge_base/raw", category: str = None) -> None: """Clean up raw repositories after embeddings are built""" logger = logging.getLogger(__name__) with open(config_path, "r") as f: config = yaml.safe_load(f) categories = [category] if category else list(config.keys()) for cat in categories: repos = config.get(cat) if not isinstance(repos, list): continue for repo in repos: repo_name = repo["name"] repo_dir = Path(base_path) / cat / repo_name if repo_dir.exists(): try: import shutil shutil.rmtree(repo_dir) logger.info(f"Cleaned up {cat}/{repo_name}") except Exception as e: logger.warning(f"Failed to clean up {cat}/{repo_name}: {e}") # Clean up empty category directories base_path_obj = Path(base_path) if base_path_obj.exists(): for cat_dir in base_path_obj.iterdir(): if cat_dir.is_dir() and not any(cat_dir.iterdir()): try: cat_dir.rmdir() logger.info(f"Cleaned up empty category directory: {cat_dir}") except Exception as e: logger.warning(f"Failed to clean up empty category directory {cat_dir}: {e}") if __name__ == "__main__": parser = argparse.ArgumentParser( description="Build the knowledge base by cloning repositories, downloading PDFs, and creating txtai embeddings." ) parser.add_argument( "--config", default="config/repositories.yml", help="Path to repository configuration file", ) parser.add_argument( "--articles-config", default=None, help="Path to PDF articles configuration file (optional)", ) parser.add_argument( "--base-path", default="knowledge_base/raw", help="Base path for repositories and PDFs", ) parser.add_argument( "--embeddings-path", default="knowledge_base/embeddings", help="Path for embeddings index", ) parser.add_argument( "--dry-run", action="store_true", help="Show what would be done without making changes", ) parser.add_argument("--category", help="Process only a specific category") parser.add_argument( "--force-update", action="store_true", help="Update repositories and re-download PDFs if they already exist", ) parser.add_argument( "--repo", default=None, help="Limit processing to one repository by name", ) parser.add_argument( "--dirty", action="store_true", help="Leave the raw repos and PDFs in place after embeddings are built", ) parser.add_argument( "--summaries-only", action="store_true", default=False, help=( "Clone/update repositories and populate the summary cache, then exit " "without building the embeddings index. Intended for parallel " "cluster pre-warming (use with --repo to process one repo per node)." ), ) parser.add_argument( "--summaries", dest="summaries", action="store_true", default=DEFAULT_SUMMARIES_ENABLED, help="Generate document summaries for each document (local, custom OpenAI-compatible, or Anthropic)", ) parser.add_argument( "--no-summaries", dest="summaries", action="store_false", help="Disable summary generation even if ENABLE_DOC_SUMMARIES is set", ) parser.add_argument( "--use-cached-ocr-only", action="store_true", default=False, help=( "Only use cached OCR Markdown for PDFs. Missing cache entries are marked as deferred " "needs_ocr instead of trying live OCR." ), ) args = parser.parse_args() if "build_pipeline" not in globals(): def print_pipeline_summary(all_failures: dict, dry_run: bool = False) -> None: logger = logging.getLogger(__name__) prefix = "[DRY RUN] " if dry_run else "" logger.info("=" * 60) logger.info(f"{prefix}KNOWLEDGE BASE PIPELINE SUMMARY") logger.info("=" * 60) repo_failures = all_failures.get("repos", {}) if repo_failures: logger.info("\n📁 REPOSITORIES:") if repo_failures.get("successful_clones"): logger.info(f" ✅ Successfully cloned: {len(repo_failures['successful_clones'])}") if repo_failures.get("successful_updates"): logger.info(f" 🔄 Successfully updated: {len(repo_failures['successful_updates'])}") if repo_failures.get("skipped_existing"): logger.info(f" ⏭️ Skipped (already exists): {len(repo_failures['skipped_existing'])}") if repo_failures.get("failed_clones"): logger.info(f" ❌ Failed to clone: {len(repo_failures['failed_clones'])}") if repo_failures.get("failed_updates"): logger.info(f" ❌ Failed to update: {len(repo_failures['failed_updates'])}") article_failures = all_failures.get("articles", {}) if article_failures: logger.info("\n📚 PDF ARTICLES:") if article_failures.get("successful_downloads"): logger.info(f" ✅ Successfully downloaded: {len(article_failures['successful_downloads'])}") if article_failures.get("skipped_existing"): logger.info(f" ⏭️ Skipped (already exists): {len(article_failures['skipped_existing'])}") if article_failures.get("failed_downloads"): logger.info(f" ❌ Failed to download: {len(article_failures['failed_downloads'])}") indexing_failures = all_failures.get("indexing", {}) or {} if indexing_failures: logger.info("\n🔍 INDEXING & EMBEDDING:") logger.info( f" ✅ Successfully indexed text files: {indexing_failures.get('successful_text_files', 0)}" ) logger.info(f" ✅ Successfully indexed PDF files: {indexing_failures.get('successful_pdf_files', 0)}") if indexing_failures.get("indexed_document_count") is not None: logger.info(f" 📄 Indexed document chunks: {indexing_failures.get('indexed_document_count', 0)}") if indexing_failures.get("text_file_candidates") or indexing_failures.get("pdf_file_candidates"): logger.info( " 🔎 Source candidates — text: " f"{indexing_failures.get('text_file_candidates', 0)}, " f"PDF: {indexing_failures.get('pdf_file_candidates', 0)}" ) if indexing_failures.get("failed_text_files"): logger.info(f" ❌ Failed text files: {len(indexing_failures['failed_text_files'])}") if indexing_failures.get("skipped_articles"): logger.info(f" ⏭️ Skipped articles: {len(indexing_failures['skipped_articles'])}") if indexing_failures.get("failed_pdf_files"): logger.info(f" ❌ Failed PDF files: {len(indexing_failures['failed_pdf_files'])}") if indexing_failures.get("deferred_pdf_files"): logger.info(f" ⏸️ Deferred PDF files (needs OCR): {len(indexing_failures['deferred_pdf_files'])}") if indexing_failures.get("fatal_errors"): for err in indexing_failures["fatal_errors"]: logger.error(f" 🚫 Fatal: {err}") summary_stats = indexing_failures.get("summary_stats") if summary_stats: logger.info("\n🧠 LLM SUMMARIES:") configured = summary_stats.get("configured", False) enabled = summary_stats.get("enabled", False) if configured: status_icon = "✅" if enabled else "⛔️" logger.info(f" {status_icon} Summaries enabled: {enabled}") else: logger.info(" ⏸️ Summaries disabled (flag not set)") if enabled: logger.info(f" 📨 Summary requests: {summary_stats.get('requests', 0)}") logger.info(f" 📝 Summaries returned: {summary_stats.get('responses', 0)}") if summary_stats.get("cached_hits"): logger.info(f" ♻️ Cached hits: {summary_stats.get('cached_hits', 0)}") if summary_stats.get("failures"): logger.info(f" ❌ Summary failures: {summary_stats.get('failures', 0)}") logger.info(f" 📦 Summary documents added: {summary_stats.get('documents_added', 0)}") elif configured: logger.info( " ℹ️ Summaries configured but disabled " "(check local, custom OpenAI-compatible, or Anthropic summary configuration)." ) total_failures = ( sum(len(v) for k, v in indexing_failures.items() if k.startswith("failed")) + len(indexing_failures.get("fatal_errors", [])) + len(article_failures.get("failed_downloads", [])) ) logger.info("\n" + "=" * 60) if total_failures == 0: logger.info(f"{prefix}✅ PIPELINE COMPLETED SUCCESSFULLY - No failures detected!") else: logger.info(f"{prefix}⚠️ PIPELINE COMPLETED WITH {total_failures} FAILURES") logger.info("=" * 60) def build_pipeline( config_path: str, articles_config_path: str = None, base_path: str = "knowledge_base/raw", embeddings_path: str = "knowledge_base/embeddings", dry_run: bool = False, category: str = None, force_update: bool = False, repo_filter: Optional[str] = None, dirty: bool = False, summaries: bool = False, summaries_only: bool = False, use_cached_ocr_only: bool = False, ) -> None: logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s") logger = logging.getLogger(__name__) all_failures = {"repos": {}, "articles": {}, "indexing": {}} summary_generator = None if summaries or summaries_only: cache_dir = Path(embeddings_path).parent / "cache" / "summaries" summary_generator = SummaryGenerator(cache_dir=cache_dir, enabled=True) emit_progress(5, stage="start", detail="Cloning repositories") all_failures["repos"] = clone_repositories( config_path, base_path, dry_run, category, force_update, repo_filter ) emit_progress(25, stage="repos_done", detail="Repositories cloned/updated") # --summaries-only: walk files to warm the summary cache, then exit # without writing the embeddings index. For parallel cluster # pre-warming; pair with --repo to process one repo per node. if summaries_only and summary_generator is not None: if articles_config_path and os.path.exists(articles_config_path): emit_progress(30, stage="articles", detail="Downloading PDF articles") all_failures["articles"] = download_pdf_articles( articles_config_path, base_path, dry_run, category, force_update ) emit_progress(50, stage="articles_done", detail="PDF articles downloaded") logger.info( "--summaries-only: warming summary cache for %s", repo_filter or "all repos", ) all_failures["indexing"] = build_txtai_index( config_path, articles_config_path, base_path, embeddings_path, dry_run=True, # don't write an index; cache writes happen regardless category=category, summary_generator=summary_generator, repo_filter=repo_filter, use_cached_ocr_only=use_cached_ocr_only, ) print_pipeline_summary(all_failures, dry_run) if env_truthy("NB_STRICT_SUMMARY_WARM"): indexing_failures = all_failures.get("indexing") or {} failed_texts = indexing_failures.get("failed_text_files") or [] failed_pdfs = indexing_failures.get("failed_pdf_files") or [] successful_texts = indexing_failures.get("successful_text_files", 0) successful_pdfs = indexing_failures.get("successful_pdf_files", 0) if failed_texts or failed_pdfs or (successful_texts + successful_pdfs) == 0: logger.error( "Strict summary warm failed: %s text failures, %s PDF failures, %s successful documents.", len(failed_texts), len(failed_pdfs), successful_texts + successful_pdfs, ) sys.exit(1) emit_progress(100, stage="done", detail="Summary cache warmed") return if articles_config_path and os.path.exists(articles_config_path): emit_progress(30, stage="articles", detail="Downloading PDF articles") all_failures["articles"] = download_pdf_articles( articles_config_path, base_path, dry_run, category, force_update ) emit_progress(50, stage="articles_done", detail="PDF articles downloaded") all_failures["indexing"] = build_txtai_index( config_path, articles_config_path, base_path, embeddings_path, dry_run, category, summary_generator, repo_filter, use_cached_ocr_only=use_cached_ocr_only, ) emit_progress(85, stage="indexing_done", detail="Indexing completed") if not dirty and not dry_run: cleanup_raw_repositories(config_path, base_path, category) if articles_config_path and os.path.exists(articles_config_path): cleanup_pdf_articles(articles_config_path, base_path, category) emit_progress(95, stage="cleanup", detail="Cleaning up raw files") print_pipeline_summary(all_failures, dry_run) if not dry_run: indexing_failures = all_failures.get("indexing") or {} fatal_errors = indexing_failures.get("fatal_errors") or [] if fatal_errors: for msg in fatal_errors: logger.error(msg) sys.exit(1) failed_texts = indexing_failures.get("failed_text_files") or [] failed_pdfs = indexing_failures.get("failed_pdf_files") or [] if failed_texts or failed_pdfs: logger.warning( "Indexing completed with failures. See log output for details on the affected files." ) emit_progress(100, stage="done", detail="Pipeline finished") build_pipeline( config_path=args.config, articles_config_path=args.articles_config, base_path=args.base_path, embeddings_path=args.embeddings_path, dry_run=args.dry_run, category=args.category, force_update=args.force_update, repo_filter=args.repo, dirty=args.dirty, summaries=args.summaries, summaries_only=args.summaries_only, use_cached_ocr_only=args.use_cached_ocr_only, )
"""
This script processes all repositories in the config/repositories.yml file.
Orchestrates the full knowledge base build pipeline (cloning, direct txtai indexing).
"""
import os
import sys
# Allow duplicate OpenMP runtime loading when not explicitly configured. This mirrors
# the CLI behavior so builds don't abort due to multiple libomp copies.
os.environ["KMP_DUPLICATE_LIB_OK"] = "TRUE"
import yaml
import subprocess
from pathlib import Path
import logging
import argparse
import requests
import json
import time
import threading
from typing import Optional, Set, Dict
try:
from dotenv import load_dotenv
except Exception:
load_dotenv = None
from chunky import ChunkPipeline, ChunkerConfig, Document
from nancy_brain.chunking import strip_chunk_suffix
from nancy_brain.pdf_ocr import extract_pdf_markdown, get_pdf_ocr_backend_status
from nancy_brain.summarization import SummaryGenerator
from scripts.script_utils import is_full_commit_sha as _is_full_commit_sha
# Optional imports
try:
from nb4llm import convert_ipynb_to_txt
except ImportError:
convert_ipynb_to_txt = None
# Load .env overrides early so subsequent imports see them
if load_dotenv is not None:
try:
script_root = Path(__file__).resolve().parent.parent
env_path = script_root / "config" / ".env"
if env_path.exists():
load_dotenv(env_path)
except Exception:
pass
# Ensure OpenMP duplicate setting persists after dotenv loads
os.environ["KMP_DUPLICATE_LIB_OK"] = "TRUE"
# PDF exclusion & thresholds
MIN_PDF_BYTES = int(os.environ.get("MIN_PDF_BYTES", "5000")) # skip tiny/image-y PDFs
MIN_PDF_TEXT_CHARS = int(os.environ.get("MIN_PDF_TEXT_CHARS", "500")) # require meaningful text
DEFAULT_EXCLUDE_PDF_SUBSTRINGS = [
"logo/",
"PointSpreadFunctions/",
"PSF_",
"PSFs_",
"PSF-",
"PSF.",
"/graphics/",
"workflow-",
"Glossary.pdf",
"column-mapping.pdf",
]
ENV_EXCLUDES = [e.strip() for e in os.environ.get("PDF_EXCLUDE_SUBSTRINGS", "").split(",") if e.strip()]
EXCLUDE_PDF_SUBSTRINGS = DEFAULT_EXCLUDE_PDF_SUBSTRINGS + ENV_EXCLUDES
README_CANDIDATES = ["README.md", "README.rst", "README.txt"]
DEFAULT_SUMMARIES_ENABLED = os.environ.get("ENABLE_DOC_SUMMARIES", "false").lower() == "true"
SKIP_PDFS = os.environ.get("SKIP_PDF_PROCESSING", "").strip().lower() == "true"
TEXT_EXTENSIONS = {".py", ".md", ".txt", ".rst", ".tex", ".yaml", ".yml", ".json"}
SKIP_DIR_NAMES = {
".git",
".github",
"__pycache__",
"node_modules",
".pytest_cache",
".mypy_cache",
".idea",
".venv",
"env",
"venv",
}
# Minimum content length (stripped) to bother generating a summary.
# Override with NB_MIN_SUMMARY_CHARS environment variable.
MIN_SUMMARY_CHARS = int(os.environ.get("NB_MIN_SUMMARY_CHARS", "200"))
# File extensions for which summary generation is never useful.
# These files are still indexed via their path/doc_id.
SUMMARY_SKIP_EXTENSIONS = {
# Astronomy data
".fits",
".fit",
# Numpy/scipy
".npy",
".npz",

Xet Storage Details

Size:
71 kB
·
Xet hash:
5720f7d5d1a5367a56b2582eccaf5a1c8e675fd17f1cdb95ee6602a0863b321f

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.