Buckets:
AnonymousSpottedToad/candidate_bucket / codebase /python /AmberLee2427_nancy-brain /scripts /build_knowledge_base.py
"""
This script processes all repositories in the config/repositories.yml file.
Orchestrates the full knowledge base build pipeline (cloning, direct txtai indexing).
"""
import os
import sys
# Allow duplicate OpenMP runtime loading when not explicitly configured. This mirrors
# the CLI behavior so builds don't abort due to multiple libomp copies.
os.environ["KMP_DUPLICATE_LIB_OK"] = "TRUE"
import yaml
import subprocess
from pathlib import Path
import logging
import argparse
import requests
import json
import time
import threading
from typing import Optional, Set, Dict
try:
from dotenv import load_dotenv
except Exception:
load_dotenv = None
from chunky import ChunkPipeline, ChunkerConfig, Document
from nancy_brain.chunking import strip_chunk_suffix
from nancy_brain.pdf_ocr import extract_pdf_markdown, get_pdf_ocr_backend_status
from nancy_brain.summarization import SummaryGenerator
from scripts.script_utils import is_full_commit_sha as _is_full_commit_sha
# Optional imports
try:
from nb4llm import convert_ipynb_to_txt
except ImportError:
convert_ipynb_to_txt = None
# Load .env overrides early so subsequent imports see them
if load_dotenv is not None:
try:
script_root = Path(__file__).resolve().parent.parent
env_path = script_root / "config" / ".env"
if env_path.exists():
load_dotenv(env_path)
except Exception:
pass
# Ensure OpenMP duplicate setting persists after dotenv loads
os.environ["KMP_DUPLICATE_LIB_OK"] = "TRUE"
# PDF exclusion & thresholds
MIN_PDF_BYTES = int(os.environ.get("MIN_PDF_BYTES", "5000")) # skip tiny/image-y PDFs
MIN_PDF_TEXT_CHARS = int(os.environ.get("MIN_PDF_TEXT_CHARS", "500")) # require meaningful text
DEFAULT_EXCLUDE_PDF_SUBSTRINGS = [
"logo/",
"PointSpreadFunctions/",
"PSF_",
"PSFs_",
"PSF-",
"PSF.",
"/graphics/",
"workflow-",
"Glossary.pdf",
"column-mapping.pdf",
]
ENV_EXCLUDES = [e.strip() for e in os.environ.get("PDF_EXCLUDE_SUBSTRINGS", "").split(",") if e.strip()]
EXCLUDE_PDF_SUBSTRINGS = DEFAULT_EXCLUDE_PDF_SUBSTRINGS + ENV_EXCLUDES
README_CANDIDATES = ["README.md", "README.rst", "README.txt"]
DEFAULT_SUMMARIES_ENABLED = os.environ.get("ENABLE_DOC_SUMMARIES", "false").lower() == "true"
SKIP_PDFS = os.environ.get("SKIP_PDF_PROCESSING", "").strip().lower() == "true"
TEXT_EXTENSIONS = {".py", ".md", ".txt", ".rst", ".tex", ".yaml", ".yml", ".json"}
SKIP_DIR_NAMES = {
".git",
".github",
"__pycache__",
"node_modules",
".pytest_cache",
".mypy_cache",
".idea",
".venv",
"env",
"venv",
}
# Minimum content length (stripped) to bother generating a summary.
# Override with NB_MIN_SUMMARY_CHARS environment variable.
MIN_SUMMARY_CHARS = int(os.environ.get("NB_MIN_SUMMARY_CHARS", "200"))
# File extensions for which summary generation is never useful.
# These files are still indexed via their path/doc_id.
SUMMARY_SKIP_EXTENSIONS = {
# Astronomy data
".fits",
".fit",
# Numpy/scipy
".npy",
".npz",
# Pickle
".pkl",
".pickle",
# Tabular
".dat",
".csv",
".parquet",
# HDF5
".hdf5",
".h5",
# MATLAB / NIfTI
".mat",
".nii",
# Archives
".gz",
".bz2",
".xz",
".zip",
".tar",
# Compiled objects
".so",
".o",
".a",
".dylib",
".dll",
# Images
".png",
".jpg",
".jpeg",
".gif",
".svg",
".ico",
".woff",
".woff2",
}
def env_truthy(name: str) -> bool:
return os.environ.get(name, "").strip().lower() in {"1", "true", "yes", "on"}
def is_excluded_pdf(path: str) -> bool:
p = str(path)
return any(token in p for token in EXCLUDE_PDF_SUBSTRINGS)
def load_repo_readme(repo_dir: Path) -> Optional[dict]:
for candidate in README_CANDIDATES:
readme_path = repo_dir / candidate
if not readme_path.exists():
continue
try:
text = readme_path.read_text(encoding="utf-8")
except Exception:
continue
if text.strip():
try:
relative = str(readme_path.relative_to(repo_dir))
except Exception:
relative = str(readme_path.name)
return {"content": text, "path": relative}
return None
def collect_repo_files(repo_dir: Path) -> tuple[list[Path], list[Path]]:
text_files: list[Path] = []
pdf_files: list[Path] = []
for root, dirs, files in os.walk(repo_dir):
dirs[:] = [d for d in dirs if d not in SKIP_DIR_NAMES]
for filename in files:
path = Path(root) / filename
suffix = path.suffix.lower()
if suffix in TEXT_EXTENSIONS or filename.endswith(".nb.txt"):
text_files.append(path)
elif suffix == ".pdf" and not SKIP_PDFS:
pdf_files.append(path)
return text_files, pdf_files
def emit_progress(percent: int, stage: str = "", detail: str = ""):
"""Emit a JSON progress marker line to stdout for UIs to parse.
Lines are prefixed with PROGRESS_JSON: to make them easy to detect in logs.
"""
try:
payload = {"percent": int(percent), "stage": stage, "detail": detail}
print("PROGRESS_JSON: " + json.dumps(payload), flush=True)
except Exception:
# Never fail the build due to progress emission
pass
# PDF download headers
PDF_REQUEST_HEADERS = {
"User-Agent": "nancy-brain-kb-builder/0.1 (+https://example.local)",
"Accept": "application/pdf,application/octet-stream;q=0.9,*/*;q=0.8",
}
def process_pdf_with_ocr(pdf_path, cache_dir=None, use_cached_ocr_only: bool = False):
logger = logging.getLogger(__name__)
preferred_backend = "skip" if use_cached_ocr_only else None
ocr_result = extract_pdf_markdown(pdf_path, cache_dir=cache_dir, preferred_backend=preferred_backend)
content = (ocr_result.markdown or "").strip()
if not content:
if ocr_result.needs_ocr:
logger.info("PDF OCR deferred for %s: cache miss and OCR worker unavailable", pdf_path)
elif ocr_result.warning:
logger.warning("PDF OCR failed for %s: %s", pdf_path, ocr_result.warning)
return None, ocr_result, False
if len(content) < MIN_PDF_TEXT_CHARS:
logger.debug(
"Discarding PDF %s (OCR markdown chars %s < %s)",
pdf_path,
len(content),
MIN_PDF_TEXT_CHARS,
)
return None, ocr_result, False
return content, ocr_result, True
# --- Utility function (unchanged from original) ---
def get_file_type_category(doc_id: str) -> str:
base_id = strip_chunk_suffix(doc_id)
path = Path(base_id)
code_extensions = {
".py",
".js",
".ts",
".cpp",
".java",
".go",
".rs",
".c",
".h",
".css",
".scss",
".jsx",
".tsx",
}
if path.suffix in code_extensions:
return "code"
if ".nb" in path.suffixes or ".nb.txt" in str(path):
return "mixed"
config_extensions = {".json", ".yaml", ".yml", ".toml", ".ini"}
if path.suffix in config_extensions:
return "mixed"
mixed_extensions = {".md", ".rst"}
if path.suffix in mixed_extensions:
return "mixed"
return "docs"
# --- Updated download function with headers ---
def download_pdf_articles(
config_path: str,
base_path: str = "knowledge_base/raw",
dry_run: bool = False,
category: str = None,
force_update: bool = False,
) -> dict:
logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
logger = logging.getLogger(__name__)
with open(config_path, "r") as f:
config = yaml.safe_load(f)
session = requests.Session()
session.headers.update(PDF_REQUEST_HEADERS)
session.max_redirects = 15
categories = [category] if category else list(config.keys())
failures = {
"failed_downloads": [],
"skipped_existing": [],
"successful_downloads": [],
}
for cat in categories:
articles = config.get(cat)
if not isinstance(articles, list):
continue
for article in articles:
article_name = article["name"]
article_url = article["url"]
dest_dir = Path(base_path) / cat
dest_file = dest_dir / f"{article_name}.pdf"
if dest_file.exists() and not force_update:
logger.info(f"Article {cat}/{article_name}.pdf already exists, skipping.")
failures["skipped_existing"].append(f"{cat}/{article_name}")
continue
logger.info(f"Downloading {article_name} from {article_url} to {dest_file}...")
if dry_run:
logger.info(f"[DRY RUN] Would download {article_url} to {dest_file}")
continue
dest_dir.mkdir(parents=True, exist_ok=True)
try:
resp = session.get(article_url, timeout=45, allow_redirects=True)
resp.raise_for_status()
ct = resp.headers.get("Content-Type", "")
if "text/html" in ct.lower():
raise RuntimeError(f"Expected PDF got HTML (content-type={ct})")
if len(resp.content) < MIN_PDF_BYTES:
raise RuntimeError(f"PDF too small ({len(resp.content)} bytes < {MIN_PDF_BYTES})")
with open(dest_file, "wb") as f:
f.write(resp.content)
logger.info(f"Successfully downloaded {article_name}")
failures["successful_downloads"].append(f"{cat}/{article_name}")
except requests.TooManyRedirects as e:
logger.error(f"Failed to download {article_name}: Redirect loop ({e})")
failures["failed_downloads"].append(f"{cat}/{article_name}: Redirect loop")
except Exception as e:
if dest_file.exists():
try:
dest_file.unlink()
except Exception:
pass
logger.error(f"Failed to download {article_name}: {e}")
failures["failed_downloads"].append(f"{cat}/{article_name}: {str(e)}")
return failures
# --- Clone repositories (original) ---
def clone_repositories(
config_path: str,
base_path: str = "knowledge_base/raw",
dry_run: bool = False,
category: str = None,
force_update: bool = False,
repo_filter: Optional[str] = None,
) -> dict:
logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
logger = logging.getLogger(__name__)
with open(config_path, "r") as f:
config = yaml.safe_load(f)
categories = [category] if category else list(config.keys())
failures = {
"failed_clones": [],
"failed_updates": [],
"skipped_existing": [],
"successful_clones": [],
"successful_updates": [],
}
for cat in categories:
repos = config.get(cat)
if not isinstance(repos, list):
continue
for repo in repos:
repo_name = repo["name"]
if repo_filter and repo_name != repo_filter:
continue
repo_url = repo["url"]
ref = repo.get("ref")
dest_dir = Path(base_path) / cat / repo_name
if dest_dir.exists():
if force_update:
logger.info(f"Repository {cat}/{repo_name} exists. Updating...")
if dry_run:
logger.info(f"[DRY RUN] Would update {cat}/{repo_name}")
continue
try:
if ref:
if _is_full_commit_sha(ref):
fetch_cmd = ["git", "fetch", "origin", ref]
else:
fetch_cmd = ["git", "fetch", "--depth", "1", "origin", ref]
subprocess.run(
fetch_cmd,
cwd=str(dest_dir),
check=True,
capture_output=True,
text=True,
)
subprocess.run(
["git", "checkout", ref],
cwd=str(dest_dir),
check=True,
capture_output=True,
text=True,
)
else:
subprocess.run(
["git", "fetch", "--all"],
cwd=str(dest_dir),
check=True,
capture_output=True,
text=True,
)
result = subprocess.run(
["git", "rev-parse", "--abbrev-ref", "HEAD"],
cwd=str(dest_dir),
capture_output=True,
text=True,
)
current_branch = result.stdout.strip()
subprocess.run(
["git", "pull", "origin", current_branch],
cwd=str(dest_dir),
check=True,
capture_output=True,
text=True,
)
failures["successful_updates"].append(f"{cat}/{repo_name}")
except subprocess.CalledProcessError as e:
failures["failed_updates"].append(f"{cat}/{repo_name}: {e.stderr}")
else:
logger.info(f"Repository {cat}/{repo_name} already exists, skipping.")
failures["skipped_existing"].append(f"{cat}/{repo_name}")
else:
logger.info(f"Cloning {repo_name} from {repo_url} into {dest_dir}...")
if dry_run:
logger.info(f"[DRY RUN] Would clone {repo_url} into {dest_dir}")
continue
dest_dir.parent.mkdir(parents=True, exist_ok=True)
try:
if ref:
if _is_full_commit_sha(ref):
clone_cmd = ["git", "clone", repo_url, str(dest_dir)]
else:
clone_cmd = ["git", "clone", "--branch", ref, "--depth", "1", repo_url, str(dest_dir)]
else:
clone_cmd = ["git", "clone", "--depth", "1", repo_url, str(dest_dir)]
subprocess.run(clone_cmd, check=True, capture_output=True, text=True)
if ref and _is_full_commit_sha(ref):
subprocess.run(
["git", "checkout", ref],
cwd=str(dest_dir),
check=True,
capture_output=True,
text=True,
)
failures["successful_clones"].append(f"{cat}/{repo_name}")
except subprocess.CalledProcessError as e:
failures["failed_clones"].append(f"{cat}/{repo_name}: {e.stderr}")
return failures
# --- Build index ---
def build_txtai_index(
config_path: str,
articles_config_path: str = None,
base_path: str = "knowledge_base/raw",
embeddings_path: str = "knowledge_base/embeddings",
dry_run: bool = False,
category: str = None,
summary_generator: Optional[SummaryGenerator] = None,
repo_filter: Optional[str] = None,
use_cached_ocr_only: bool = False,
) -> dict:
logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
logger = logging.getLogger(__name__)
try:
from txtai.embeddings import Embeddings
except ImportError:
logger.error("txtai not available. Please install with: pip install txtai")
return {
"failed_text_files": [],
"failed_pdf_files": [],
"failed_notebook_conversions": [],
"deferred_pdf_files": [],
"successful_text_files": 0,
"successful_pdf_files": 0,
"successful_notebook_conversions": 0,
"skipped_repositories": [],
"skipped_articles": [],
}
use_dual_embedding = os.environ.get("USE_DUAL_EMBEDDING", "true").lower() == "true"
code_model = os.environ.get("CODE_EMBEDDING_MODEL", "microsoft/codebert-base")
logger.info(f"Dual embedding enabled: {use_dual_embedding}")
if use_dual_embedding:
logger.info(f"Code model: {code_model}")
if SKIP_PDFS:
logger.info("SKIP_PDF_PROCESSING enabled; PDF documents will be ignored during build.")
pdf_ocr_status = get_pdf_ocr_backend_status()
if not SKIP_PDFS:
if pdf_ocr_status.available:
logger.info("✅ PDF OCR backend selected: %s (%s)", pdf_ocr_status.name, pdf_ocr_status.model)
else:
logger.warning(
"PDF OCR unavailable; PDFs will be skipped unless OCR extras are installed: %s", pdf_ocr_status.reason
)
try:
with open(config_path, "r") as f:
config = yaml.safe_load(f)
except Exception as e:
logger.error(f"Failed to read repository config: {config_path} ({e})")
return {
"failed_text_files": [],
"failed_pdf_files": [],
"failed_notebook_conversions": [],
"deferred_pdf_files": [],
"successful_text_files": 0,
"successful_pdf_files": 0,
"successful_notebook_conversions": 0,
"skipped_repositories": [],
"skipped_articles": [],
"validation_errors": [f"Failed to read config: {config_path}: {e}"],
}
# Validate repository config early to give clear feedback
try:
from nancy_brain.config_validation import validate_repositories_config, validate_articles_config
ok, errors = validate_repositories_config(config or {})
if not ok:
logger.error("repositories.yml validation failed")
return {
"failed_text_files": [],
"failed_pdf_files": [],
"failed_notebook_conversions": [],
"deferred_pdf_files": [],
"successful_text_files": 0,
"successful_pdf_files": 0,
"successful_notebook_conversions": 0,
"skipped_repositories": [],
"skipped_articles": [],
"validation_errors": errors,
}
except Exception:
# If validation helpers are unavailable for any reason, continue without failing
pass
articles_config = {}
if articles_config_path and os.path.exists(articles_config_path):
with open(articles_config_path, "r") as f:
articles_config = yaml.safe_load(f)
logger.info(f"Loaded articles configuration from {articles_config_path}")
# Validate articles config if validator available
try:
ok2, errors2 = validate_articles_config(articles_config or {})
if not ok2:
logger.error("articles.yml validation failed")
return {
"failed_text_files": [],
"failed_pdf_files": [],
"failed_notebook_conversions": [],
"deferred_pdf_files": [],
"successful_text_files": 0,
"successful_pdf_files": 0,
"successful_notebook_conversions": 0,
"skipped_repositories": [],
"skipped_articles": [],
"validation_errors": errors2,
}
except Exception:
pass
embeddings_dir = Path(embeddings_path)
embeddings_dir.mkdir(parents=True, exist_ok=True)
text_model = os.environ.get("NB_TEXT_EMBEDDING_MODEL", "sentence-transformers/all-MiniLM-L6-v2")
logger.info(f"Loading general embedding model: {text_model}")
model_start = time.time()
try:
general_embeddings = Embeddings(
{
"path": text_model,
"content": True,
"backend": "faiss",
}
)
except Exception as exc:
logger.error(f"Failed to load general embedding model '{text_model}': {exc}")
logger.error(
"If this is the first run, ensure the model is reachable or set NB_TEXT_EMBEDDING_MODEL"
" to a smaller/local SentenceTransformer."
)
raise
logger.info(f"General embedding model ready in {time.time() - model_start:.2f}s")
code_embeddings = None
if use_dual_embedding:
code_model_override = os.environ.get("NB_CODE_EMBEDDING_MODEL", code_model)
logger.info(f"Loading code embedding model: {code_model_override}")
code_start = time.time()
try:
code_embeddings = Embeddings({"path": code_model_override, "content": True, "backend": "faiss"})
except Exception as exc:
logger.error(f"Failed to load code embedding model '{code_model_override}': {exc}")
logger.error(
"Disable dual embeddings with USE_DUAL_EMBEDDING=false or set NB_CODE_EMBEDDING_MODEL"
" to a lighter model."
)
raise
logger.info(f"Code embedding model ready in {time.time() - code_start:.2f}s")
try:
with open("config/index_weights.yaml", "r") as f:
ext_weights = yaml.safe_load(f)
except Exception:
ext_weights = {}
categories = [category] if category else list(config.keys())
# Count total repositories for progress tracking
total_repos = 0
for cat in categories:
repos_list = config.get(cat)
if isinstance(repos_list, list):
if repo_filter:
total_repos += sum(1 for repo in repos_list if repo.get("name") == repo_filter)
else:
total_repos += len(repos_list)
repos_processed = 0
documents = []
pipeline = ChunkPipeline()
pdf_ocr_cache_dir = Path(base_path).parent / "cache" / "pdf_ocr"
chunk_config = ChunkerConfig(
max_chars=int(os.environ.get("CHUNKY_MAX_CHARS", "1000")),
lines_per_chunk=int(os.environ.get("CHUNKY_LINES_PER_CHUNK", "40")),
line_overlap=int(os.environ.get("CHUNKY_LINE_OVERLAP", "5")),
)
pdf_status = {} # doc_id -> {status, size, chars, method}
failures = {
"failed_text_files": [],
"failed_pdf_files": [],
"failed_notebook_conversions": [],
"deferred_pdf_files": [],
"successful_text_files": 0,
"successful_pdf_files": 0,
"successful_notebook_conversions": 0,
"skipped_repositories": [],
"skipped_articles": [],
"skipped_low_value": [],
"fatal_errors": [],
}
text_candidate_count = 0
pdf_candidate_count = 0
summary_documents = []
summary_enabled = False
if summary_generator is not None:
summary_enabled = bool(getattr(summary_generator, "enabled", True))
summary_stats = {
"configured": summary_generator is not None,
"enabled": summary_enabled,
"requests": 0,
"responses": 0,
"cached_hits": 0,
"failures": 0,
"documents_added": 0,
}
auto_model_weights: Dict[str, float] = {}
summarized_docs: Set[str] = set()
repo_readme_cache: Dict[Path, Optional[dict]] = {}
def should_skip_summary(doc_id: str, content: str, file_path: Optional[Path]) -> bool:
stripped = content.strip()
if len(stripped) < MIN_SUMMARY_CHARS:
logger.debug(
"Skipping summary for %s: content too short (%d chars < %d)",
doc_id,
len(stripped),
MIN_SUMMARY_CHARS,
)
return True
if file_path is not None and file_path.suffix.lower() in SUMMARY_SKIP_EXTENSIONS:
logger.debug(
"Skipping summary for %s: extension %s in SUMMARY_SKIP_EXTENSIONS",
doc_id,
file_path.suffix.lower(),
)
return True
return False
for cat in categories:
repos = config.get(cat)
if not isinstance(repos, list):
continue
for repo in repos:
repo_name = repo["name"]
if repo_filter and repo_name != repo_filter:
continue
# Emit repo progress (50-85% range)
repos_processed += 1
pct = 50 + int(((repos_processed - 0.5) / max(1, total_repos)) * 35)
emit_progress(pct, stage="indexing", detail=f"Indexing {repo_name} ({repos_processed}/{total_repos})")
repo_dir = Path(base_path) / cat / repo_name
if not repo_dir.exists():
failures["skipped_repositories"].append(f"{cat}/{repo_name}")
continue
repo_readme_content = None
repo_readme_path = None
if summary_generator is not None:
if repo_dir not in repo_readme_cache:
repo_readme_cache[repo_dir] = load_repo_readme(repo_dir)
repo_readme_info = repo_readme_cache.get(repo_dir) or {}
repo_readme_content = repo_readme_info.get("content")
repo_readme_path = repo_readme_info.get("path")
if convert_ipynb_to_txt is not None:
for ipynb_file in repo_dir.rglob("*.ipynb"):
nb_txt_file = ipynb_file.with_suffix(".nb.txt")
if not nb_txt_file.exists() or ipynb_file.stat().st_mtime > nb_txt_file.stat().st_mtime:
try:
convert_ipynb_to_txt(str(ipynb_file), str(nb_txt_file))
failures["successful_notebook_conversions"] += 1
except Exception as e:
failures["failed_notebook_conversions"].append(f"{ipynb_file}: {str(e)}")
text_files, pdf_files = collect_repo_files(repo_dir)
if SKIP_PDFS:
pdf_files = []
else:
filtered_pdf_files = []
for pf in pdf_files:
try:
if is_excluded_pdf(pf):
pdf_status[str(pf)] = {
"status": "excluded",
"reason": "pattern",
"size": pf.stat().st_size if pf.exists() else 0,
}
continue
size = pf.stat().st_size
if size < MIN_PDF_BYTES:
pdf_status[str(pf)] = {
"status": "skipped",
"reason": f"small({size})",
"size": size,
}
continue
filtered_pdf_files.append(pf)
except Exception:
continue
pdf_files = filtered_pdf_files
text_files = [f for f in text_files if "docs/build" not in str(f) and not str(f).endswith(".rst.txt")]
if (
os.environ.get("NB_SCAN_LOG", "false").lower() == "true"
or os.environ.get("NB_PER_FILE_LOG", "false").lower() == "true"
):
logger.info(
f"Repo {repo_name}: {len(text_files)} candidate text files, {len(pdf_files)} candidate PDFs"
)
# Use text extraction helpers for .rst and .tex files when available
try:
from scripts.text_extract import extract_text_from_rst, extract_text_from_tex
except Exception:
extract_text_from_rst = None
extract_text_from_tex = None
for file_path in text_files:
try:
text_candidate_count += 1
logger.info(f"Reading file {file_path.relative_to(repo_dir)}")
with open(file_path, "r", encoding="utf-8") as f:
content = f.read()
if not content.strip():
continue
suffix = file_path.suffix.lower()
if suffix == ".rst" and extract_text_from_rst:
try:
content = extract_text_from_rst(content)
except Exception:
pass
if suffix == ".tex" and extract_text_from_tex:
try:
content = extract_text_from_tex(content)
except Exception:
pass
if not content or not content.strip():
continue
relative_path = file_path.relative_to(repo_dir)
doc_id = f"{cat}/{repo_name}/{relative_path}"
logger.info(f"Chunking file {doc_id}")
if summary_generator is not None and summary_stats["enabled"] and doc_id not in summarized_docs:
if not should_skip_summary(doc_id, content, file_path):
summary_stats["requests"] += 1
summary_payload = None
failed_due_to_exception = False
try:
summary_payload = summary_generator.summarize(
doc_id=doc_id,
content=content,
repo_name=repo_name,
repo_readme=repo_readme_content,
repo_readme_path=repo_readme_path,
repo_description=repo.get("description"),
metadata={"category": cat, "source": "repository_file"},
)
except Exception as exc:
failed_due_to_exception = True
summary_stats["failures"] += 1
logger.warning("Summary generation failed for %s: %s", doc_id, exc)
if summary_payload is not None:
summary_stats["responses"] += 1
if summary_payload.cached:
summary_stats["cached_hits"] += 1
summary_meta = {
"source_document": doc_id,
"category": cat,
"repository": repo_name,
"doc_type": "summary",
"model": summary_payload.model,
"cached": summary_payload.cached,
}
if summary_payload.repo_readme_path:
summary_meta["repo_readme_path"] = summary_payload.repo_readme_path
summary_documents.append(
(
f"summaries/{doc_id}",
summary_payload.summary,
json.dumps(summary_meta, ensure_ascii=False),
)
)
auto_model_weights[doc_id] = summary_payload.weight
summarized_docs.add(doc_id)
elif not failed_due_to_exception:
summary_stats["failures"] += 1
document = Document(path=file_path, content=content, metadata={"doc_id": doc_id})
chunks = pipeline.chunk_documents([document], config=chunk_config)
if not chunks:
logger.debug(f"Skipping low-value document: {doc_id}")
failures["skipped_low_value"].append(doc_id)
continue
for chunk in chunks:
meta = dict(chunk.metadata)
meta.setdefault("source_document", doc_id)
meta.setdefault("extension", suffix)
meta.setdefault("relative_path", str(relative_path))
meta.setdefault("repository", repo_name)
documents.append(
(
chunk.chunk_id,
chunk.text,
json.dumps(meta, ensure_ascii=False),
)
)
failures["successful_text_files"] += 1
except Exception as e:
failures["failed_text_files"].append(f"{file_path}: {str(e)}")
for pdf_path in pdf_files:
try:
pdf_candidate_count += 1
content, ocr_result, success = process_pdf_with_ocr(
pdf_path,
cache_dir=pdf_ocr_cache_dir,
use_cached_ocr_only=use_cached_ocr_only,
)
size = pdf_path.stat().st_size if pdf_path.exists() else 0
if success and content:
relative_path = pdf_path.relative_to(repo_dir)
doc_id = f"{cat}/{repo_name}/{relative_path}"
metadata = (
f"# Repository PDF\n\n"
f"- Repository: {repo_name}\n"
f"- Source: {repo['url']}\n"
f"- Path: `{relative_path}`\n"
f"- OCR backend: {ocr_result.backend}\n"
f"- OCR cache: {'hit' if ocr_result.cached else 'miss'}\n\n"
)
full_content = metadata + content
if summary_generator is not None and summary_stats["enabled"] and doc_id not in summarized_docs:
if not should_skip_summary(doc_id, full_content, pdf_path):
summary_stats["requests"] += 1
summary_payload = None
failed_due_to_exception = False
try:
summary_payload = summary_generator.summarize(
doc_id=doc_id,
content=full_content,
repo_name=repo_name,
repo_readme=repo_readme_content,
repo_readme_path=repo_readme_path,
metadata={"category": cat, "source": "repository_pdf"},
)
except Exception as exc:
failed_due_to_exception = True
summary_stats["failures"] += 1
logger.warning("Summary generation failed for %s: %s", doc_id, exc)
if summary_payload is not None:
summary_stats["responses"] += 1
if summary_payload.cached:
summary_stats["cached_hits"] += 1
summary_meta = {
"source_document": doc_id,
"category": cat,
"repository": repo_name,
"doc_type": "summary",
"model": summary_payload.model,
"cached": summary_payload.cached,
}
if summary_payload.repo_readme_path:
summary_meta["repo_readme_path"] = summary_payload.repo_readme_path
summary_documents.append(
(
f"summaries/{doc_id}",
summary_payload.summary,
json.dumps(summary_meta, ensure_ascii=False),
)
)
auto_model_weights[doc_id] = summary_payload.weight
summarized_docs.add(doc_id)
elif not failed_due_to_exception:
summary_stats["failures"] += 1
markdown_path = Path(f"{pdf_path}.ocr.md")
document = Document(path=markdown_path, content=full_content, metadata={"doc_id": doc_id})
chunks = pipeline.chunk_documents([document], config=chunk_config)
if not chunks:
logger.debug(f"Skipping low-value PDF: {doc_id}")
failures["skipped_low_value"].append(doc_id)
continue
for chunk in chunks:
meta = dict(chunk.metadata)
meta.setdefault("source_document", doc_id)
meta.setdefault("relative_path", str(relative_path))
meta.setdefault("repository", repo_name)
meta.setdefault("pdf_size_bytes", size)
meta.setdefault("source_url", repo.get("url"))
meta.setdefault("ocr_backend", ocr_result.backend)
meta.setdefault("ocr_cached", ocr_result.cached)
meta.setdefault("ocr_cache_key", ocr_result.cache_key)
documents.append(
(
chunk.chunk_id,
chunk.text,
json.dumps(meta, ensure_ascii=False),
)
)
failures["successful_pdf_files"] += 1
pdf_status[doc_id] = {
"status": "indexed",
"size": size,
"chars": len(content),
"chunk_count": len(chunks),
"backend": ocr_result.backend,
"cached": ocr_result.cached,
"cache_key": ocr_result.cache_key,
"page_count": ocr_result.page_count,
}
elif ocr_result.needs_ocr and use_cached_ocr_only:
failures["deferred_pdf_files"].append(str(pdf_path))
pdf_status[str(pdf_path)] = {
"status": "needs_ocr",
"size": size,
"backend": ocr_result.backend,
"reason": ocr_result.warning,
}
else:
reason = ocr_result.warning or "No meaningful markdown extracted"
failures["failed_pdf_files"].append(f"{pdf_path}: {reason}")
pdf_status[str(pdf_path)] = {
"status": "failed_extract",
"size": size,
"backend": ocr_result.backend,
"reason": reason,
}
except Exception as e:
failures["failed_pdf_files"].append(f"{pdf_path}: {str(e)}")
pdf_status[str(pdf_path)] = {
"status": "error",
"error": str(e)[:120],
}
article_categories = [category] if category else list(articles_config.keys())
for cat in article_categories:
articles = articles_config.get(cat)
if not isinstance(articles, list):
continue
for article in articles:
article_name = article["name"]
pdf_file = Path(base_path) / cat / f"{article_name}.pdf"
if not pdf_file.exists():
failures["skipped_articles"].append(f"{cat}/{article_name}")
continue
try:
pdf_candidate_count += 1
content, ocr_result, success = process_pdf_with_ocr(
pdf_file,
cache_dir=pdf_ocr_cache_dir,
use_cached_ocr_only=use_cached_ocr_only,
)
if success and content:
doc_id = f"journal_articles/{cat}/{article_name}"
metadata = (
f"# Journal Article\n\n"
f"- Title: {article['description']}\n"
f"- Source: {article.get('url', 'Unknown')}\n"
f"- OCR backend: {ocr_result.backend}\n"
f"- OCR cache: {'hit' if ocr_result.cached else 'miss'}\n\n"
)
full_content = metadata + content
if summary_generator is not None and summary_stats["enabled"] and doc_id not in summarized_docs:
if not should_skip_summary(doc_id, full_content, pdf_file):
summary_stats["requests"] += 1
summary_payload = None
failed_due_to_exception = False
try:
summary_payload = summary_generator.summarize(
doc_id=doc_id,
content=full_content,
repo_name=cat,
repo_readme=None,
repo_readme_path=None,
metadata={"category": cat, "source": "journal_pdf", "article": article_name},
)
except Exception as exc:
failed_due_to_exception = True
summary_stats["failures"] += 1
logger.warning("Summary generation failed for %s: %s", doc_id, exc)
if summary_payload is not None:
summary_stats["responses"] += 1
if summary_payload.cached:
summary_stats["cached_hits"] += 1
summary_meta = {
"source_document": doc_id,
"category": cat,
"doc_type": "summary",
"model": summary_payload.model,
"cached": summary_payload.cached,
}
if summary_payload.repo_readme_path:
summary_meta["repo_readme_path"] = summary_payload.repo_readme_path
summary_documents.append(
(
f"summaries/{doc_id}",
summary_payload.summary,
json.dumps(summary_meta, ensure_ascii=False),
)
)
auto_model_weights[doc_id] = summary_payload.weight
summarized_docs.add(doc_id)
elif not failed_due_to_exception:
summary_stats["failures"] += 1
markdown_path = Path(f"{pdf_file}.ocr.md")
document = Document(path=markdown_path, content=full_content, metadata={"doc_id": doc_id})
chunks = pipeline.chunk_documents([document], config=chunk_config)
if not chunks:
logger.debug(f"Skipping low-value journal PDF: {doc_id}")
failures["skipped_low_value"].append(doc_id)
continue
size_bytes = pdf_file.stat().st_size if pdf_file.exists() else 0
for chunk in chunks:
meta = dict(chunk.metadata)
meta.setdefault("source_document", doc_id)
meta.setdefault("article_name", article_name)
meta.setdefault("category", meta.get("category", "docs"))
meta.setdefault("pdf_size_bytes", size_bytes)
meta.setdefault("source_url", article.get("url"))
meta.setdefault("ocr_backend", ocr_result.backend)
meta.setdefault("ocr_cached", ocr_result.cached)
meta.setdefault("ocr_cache_key", ocr_result.cache_key)
documents.append(
(
chunk.chunk_id,
chunk.text,
json.dumps(meta, ensure_ascii=False),
)
)
failures["successful_pdf_files"] += 1
pdf_status[doc_id] = {
"status": "indexed",
"size": size_bytes,
"chars": len(content),
"chunk_count": len(chunks),
"backend": ocr_result.backend,
"cached": ocr_result.cached,
"cache_key": ocr_result.cache_key,
"page_count": ocr_result.page_count,
}
elif ocr_result.needs_ocr and use_cached_ocr_only:
failures["deferred_pdf_files"].append(str(pdf_file))
pdf_status[str(pdf_file)] = {
"status": "needs_ocr",
"backend": ocr_result.backend,
"reason": ocr_result.warning,
}
else:
reason = ocr_result.warning or "No meaningful markdown extracted"
failures["failed_pdf_files"].append(f"{pdf_file}: {reason}")
pdf_status[str(pdf_file)] = {
"status": "failed_extract",
"backend": ocr_result.backend,
"reason": reason,
}
except Exception as e:
failures["failed_pdf_files"].append(f"{pdf_file}: {str(e)}")
pdf_status[str(pdf_file)] = {
"status": "error",
"error": str(e)[:120],
}
summary_stats["documents_added"] = len(summary_documents)
failures["summary_stats"] = summary_stats
if summary_documents:
documents.extend(summary_documents)
total_indexed_documents = len(documents)
failures["indexed_document_count"] = total_indexed_documents
failures["text_file_candidates"] = text_candidate_count
failures["pdf_file_candidates"] = pdf_candidate_count
if not documents:
if not dry_run:
if text_candidate_count or pdf_candidate_count:
deferred_pdfs = failures.get("deferred_pdf_files") or []
if (
use_cached_ocr_only
and deferred_pdfs
and not failures.get("failed_text_files")
and not failures.get("failed_pdf_files")
):
logger.warning(
"No documents were indexed because %s PDFs are deferred awaiting OCR cache or worker availability.",
len(deferred_pdfs),
)
else:
failures["fatal_errors"].append(
"No documents were indexed even though source files were discovered. "
"Check for upstream processing failures (e.g., summarization errors) in the logs."
)
else:
failures["fatal_errors"].append(
"No documents were indexed because no eligible source files were found in the configured sources."
)
else:
logger.info("Dry run: no documents would be indexed.")
elif not dry_run:
logger.info(f"Indexing {len(documents)} documents...")
logger.info("Building general embeddings index… (first run may download embedding model weights)")
start = time.time()
general_embeddings.index(documents)
logger.info(f"General index built in {time.time() - start:.2f}s")
general_embeddings.save(str(embeddings_dir / "index"))
try:
index_dir = embeddings_dir / "index"
cfg_json = index_dir / "config.json"
cfg = index_dir / "config"
if cfg_json.exists() and not cfg.exists():
cfg.write_text(cfg_json.read_text(encoding="utf-8"), encoding="utf-8")
except Exception as exc:
logger.warning(f"Failed to write txtai config shim for index: {exc}")
if use_dual_embedding and code_embeddings:
logger.info("Building code embeddings index...")
cstart = time.time()
code_embeddings.index(documents)
logger.info(f"Code index built in {time.time() - cstart:.2f}s")
code_embeddings.save(str(embeddings_dir / "code_index"))
try:
code_index_dir = embeddings_dir / "code_index"
cfg_json = code_index_dir / "config.json"
cfg = code_index_dir / "config"
if cfg_json.exists() and not cfg.exists():
cfg.write_text(cfg_json.read_text(encoding="utf-8"), encoding="utf-8")
except Exception as exc:
logger.warning(f"Failed to write txtai config shim for code_index: {exc}")
results = general_embeddings.search("function", 3)
logger.info("Test search results (general model):")
for result in results:
logger.info(f" - {result['id']}: {result['text'][:100]}...")
if use_dual_embedding and code_embeddings:
code_results = code_embeddings.search("function", 3)
logger.info("Test search results (code model):")
for result in code_results:
logger.info(f" - {result['id']}: {result['text'][:100]}...")
elif not documents:
logger.warning("No documents found to index")
# Write manifest
if not dry_run:
try:
with open(embeddings_dir / "pdf_manifest.json", "w", encoding="utf-8") as mf:
json.dump(pdf_status, mf, indent=2)
logger.info(f"Wrote PDF manifest to {embeddings_dir / 'pdf_manifest.json'}")
except Exception as e:
logger.warning(f"Failed to write pdf_manifest.json: {e}")
if auto_model_weights:
try:
with open(embeddings_dir / "auto_model_weights.json", "w", encoding="utf-8") as wf:
json.dump(auto_model_weights, wf, indent=2)
logger.info(
f"Wrote suggested model weights for {len(auto_model_weights)} docs to"
f" {embeddings_dir / 'auto_model_weights.json'}"
)
except Exception as exc:
logger.warning(f"Failed to write auto_model_weights.json: {exc}")
if auto_model_weights:
failures["auto_model_weights"] = auto_model_weights
return failures
# --- Cleanup & summary functions (original) ---
def cleanup_pdf_articles(
articles_config_path: str,
base_path: str = "knowledge_base/raw",
category: str = None,
) -> None:
"""Clean up downloaded PDF articles after embeddings are built"""
logger = logging.getLogger(__name__)
with open(articles_config_path, "r") as f:
config = yaml.safe_load(f)
categories = [category] if category else list(config.keys())
for cat in categories:
articles = config.get(cat)
if not isinstance(articles, list):
continue
for article in articles:
article_name = article["name"]
pdf_file = Path(base_path) / cat / f"{article_name}.pdf"
if pdf_file.exists():
try:
pdf_file.unlink()
logger.info(f"Cleaned up PDF {cat}/{article_name}.pdf")
except Exception as e:
logger.warning(f"Failed to clean up PDF {cat}/{article_name}.pdf: {e}")
# Clean up empty category directories
base_path_obj = Path(base_path)
if base_path_obj.exists():
for cat_dir in base_path_obj.iterdir():
if cat_dir.is_dir() and not any(cat_dir.iterdir()):
try:
cat_dir.rmdir()
logger.info(f"Cleaned up empty category directory: {cat_dir}")
except Exception as e:
logger.warning(f"Failed to clean up empty category directory {cat_dir}: {e}")
def cleanup_raw_repositories(config_path: str, base_path: str = "knowledge_base/raw", category: str = None) -> None:
"""Clean up raw repositories after embeddings are built"""
logger = logging.getLogger(__name__)
with open(config_path, "r") as f:
config = yaml.safe_load(f)
categories = [category] if category else list(config.keys())
for cat in categories:
repos = config.get(cat)
if not isinstance(repos, list):
continue
for repo in repos:
repo_name = repo["name"]
repo_dir = Path(base_path) / cat / repo_name
if repo_dir.exists():
try:
import shutil
shutil.rmtree(repo_dir)
logger.info(f"Cleaned up {cat}/{repo_name}")
except Exception as e:
logger.warning(f"Failed to clean up {cat}/{repo_name}: {e}")
# Clean up empty category directories
base_path_obj = Path(base_path)
if base_path_obj.exists():
for cat_dir in base_path_obj.iterdir():
if cat_dir.is_dir() and not any(cat_dir.iterdir()):
try:
cat_dir.rmdir()
logger.info(f"Cleaned up empty category directory: {cat_dir}")
except Exception as e:
logger.warning(f"Failed to clean up empty category directory {cat_dir}: {e}")
if __name__ == "__main__":
parser = argparse.ArgumentParser(
description="Build the knowledge base by cloning repositories, downloading PDFs, and creating txtai embeddings."
)
parser.add_argument(
"--config",
default="config/repositories.yml",
help="Path to repository configuration file",
)
parser.add_argument(
"--articles-config",
default=None,
help="Path to PDF articles configuration file (optional)",
)
parser.add_argument(
"--base-path",
default="knowledge_base/raw",
help="Base path for repositories and PDFs",
)
parser.add_argument(
"--embeddings-path",
default="knowledge_base/embeddings",
help="Path for embeddings index",
)
parser.add_argument(
"--dry-run",
action="store_true",
help="Show what would be done without making changes",
)
parser.add_argument("--category", help="Process only a specific category")
parser.add_argument(
"--force-update",
action="store_true",
help="Update repositories and re-download PDFs if they already exist",
)
parser.add_argument(
"--repo",
default=None,
help="Limit processing to one repository by name",
)
parser.add_argument(
"--dirty",
action="store_true",
help="Leave the raw repos and PDFs in place after embeddings are built",
)
parser.add_argument(
"--summaries-only",
action="store_true",
default=False,
help=(
"Clone/update repositories and populate the summary cache, then exit "
"without building the embeddings index. Intended for parallel "
"cluster pre-warming (use with --repo to process one repo per node)."
),
)
parser.add_argument(
"--summaries",
dest="summaries",
action="store_true",
default=DEFAULT_SUMMARIES_ENABLED,
help="Generate document summaries for each document (local, custom OpenAI-compatible, or Anthropic)",
)
parser.add_argument(
"--no-summaries",
dest="summaries",
action="store_false",
help="Disable summary generation even if ENABLE_DOC_SUMMARIES is set",
)
parser.add_argument(
"--use-cached-ocr-only",
action="store_true",
default=False,
help=(
"Only use cached OCR Markdown for PDFs. Missing cache entries are marked as deferred "
"needs_ocr instead of trying live OCR."
),
)
args = parser.parse_args()
if "build_pipeline" not in globals():
def print_pipeline_summary(all_failures: dict, dry_run: bool = False) -> None:
logger = logging.getLogger(__name__)
prefix = "[DRY RUN] " if dry_run else ""
logger.info("=" * 60)
logger.info(f"{prefix}KNOWLEDGE BASE PIPELINE SUMMARY")
logger.info("=" * 60)
repo_failures = all_failures.get("repos", {})
if repo_failures:
logger.info("\n📁 REPOSITORIES:")
if repo_failures.get("successful_clones"):
logger.info(f" ✅ Successfully cloned: {len(repo_failures['successful_clones'])}")
if repo_failures.get("successful_updates"):
logger.info(f" 🔄 Successfully updated: {len(repo_failures['successful_updates'])}")
if repo_failures.get("skipped_existing"):
logger.info(f" ⏭️ Skipped (already exists): {len(repo_failures['skipped_existing'])}")
if repo_failures.get("failed_clones"):
logger.info(f" ❌ Failed to clone: {len(repo_failures['failed_clones'])}")
if repo_failures.get("failed_updates"):
logger.info(f" ❌ Failed to update: {len(repo_failures['failed_updates'])}")
article_failures = all_failures.get("articles", {})
if article_failures:
logger.info("\n📚 PDF ARTICLES:")
if article_failures.get("successful_downloads"):
logger.info(f" ✅ Successfully downloaded: {len(article_failures['successful_downloads'])}")
if article_failures.get("skipped_existing"):
logger.info(f" ⏭️ Skipped (already exists): {len(article_failures['skipped_existing'])}")
if article_failures.get("failed_downloads"):
logger.info(f" ❌ Failed to download: {len(article_failures['failed_downloads'])}")
indexing_failures = all_failures.get("indexing", {}) or {}
if indexing_failures:
logger.info("\n🔍 INDEXING & EMBEDDING:")
logger.info(
f" ✅ Successfully indexed text files: {indexing_failures.get('successful_text_files', 0)}"
)
logger.info(f" ✅ Successfully indexed PDF files: {indexing_failures.get('successful_pdf_files', 0)}")
if indexing_failures.get("indexed_document_count") is not None:
logger.info(f" 📄 Indexed document chunks: {indexing_failures.get('indexed_document_count', 0)}")
if indexing_failures.get("text_file_candidates") or indexing_failures.get("pdf_file_candidates"):
logger.info(
" 🔎 Source candidates — text: "
f"{indexing_failures.get('text_file_candidates', 0)}, "
f"PDF: {indexing_failures.get('pdf_file_candidates', 0)}"
)
if indexing_failures.get("failed_text_files"):
logger.info(f" ❌ Failed text files: {len(indexing_failures['failed_text_files'])}")
if indexing_failures.get("skipped_articles"):
logger.info(f" ⏭️ Skipped articles: {len(indexing_failures['skipped_articles'])}")
if indexing_failures.get("failed_pdf_files"):
logger.info(f" ❌ Failed PDF files: {len(indexing_failures['failed_pdf_files'])}")
if indexing_failures.get("deferred_pdf_files"):
logger.info(f" ⏸️ Deferred PDF files (needs OCR): {len(indexing_failures['deferred_pdf_files'])}")
if indexing_failures.get("fatal_errors"):
for err in indexing_failures["fatal_errors"]:
logger.error(f" 🚫 Fatal: {err}")
summary_stats = indexing_failures.get("summary_stats")
if summary_stats:
logger.info("\n🧠 LLM SUMMARIES:")
configured = summary_stats.get("configured", False)
enabled = summary_stats.get("enabled", False)
if configured:
status_icon = "✅" if enabled else "⛔️"
logger.info(f" {status_icon} Summaries enabled: {enabled}")
else:
logger.info(" ⏸️ Summaries disabled (flag not set)")
if enabled:
logger.info(f" 📨 Summary requests: {summary_stats.get('requests', 0)}")
logger.info(f" 📝 Summaries returned: {summary_stats.get('responses', 0)}")
if summary_stats.get("cached_hits"):
logger.info(f" ♻️ Cached hits: {summary_stats.get('cached_hits', 0)}")
if summary_stats.get("failures"):
logger.info(f" ❌ Summary failures: {summary_stats.get('failures', 0)}")
logger.info(f" 📦 Summary documents added: {summary_stats.get('documents_added', 0)}")
elif configured:
logger.info(
" ℹ️ Summaries configured but disabled "
"(check local, custom OpenAI-compatible, or Anthropic summary configuration)."
)
total_failures = (
sum(len(v) for k, v in indexing_failures.items() if k.startswith("failed"))
+ len(indexing_failures.get("fatal_errors", []))
+ len(article_failures.get("failed_downloads", []))
)
logger.info("\n" + "=" * 60)
if total_failures == 0:
logger.info(f"{prefix}✅ PIPELINE COMPLETED SUCCESSFULLY - No failures detected!")
else:
logger.info(f"{prefix}⚠️ PIPELINE COMPLETED WITH {total_failures} FAILURES")
logger.info("=" * 60)
def build_pipeline(
config_path: str,
articles_config_path: str = None,
base_path: str = "knowledge_base/raw",
embeddings_path: str = "knowledge_base/embeddings",
dry_run: bool = False,
category: str = None,
force_update: bool = False,
repo_filter: Optional[str] = None,
dirty: bool = False,
summaries: bool = False,
summaries_only: bool = False,
use_cached_ocr_only: bool = False,
) -> None:
logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
logger = logging.getLogger(__name__)
all_failures = {"repos": {}, "articles": {}, "indexing": {}}
summary_generator = None
if summaries or summaries_only:
cache_dir = Path(embeddings_path).parent / "cache" / "summaries"
summary_generator = SummaryGenerator(cache_dir=cache_dir, enabled=True)
emit_progress(5, stage="start", detail="Cloning repositories")
all_failures["repos"] = clone_repositories(
config_path, base_path, dry_run, category, force_update, repo_filter
)
emit_progress(25, stage="repos_done", detail="Repositories cloned/updated")
# --summaries-only: walk files to warm the summary cache, then exit
# without writing the embeddings index. For parallel cluster
# pre-warming; pair with --repo to process one repo per node.
if summaries_only and summary_generator is not None:
if articles_config_path and os.path.exists(articles_config_path):
emit_progress(30, stage="articles", detail="Downloading PDF articles")
all_failures["articles"] = download_pdf_articles(
articles_config_path, base_path, dry_run, category, force_update
)
emit_progress(50, stage="articles_done", detail="PDF articles downloaded")
logger.info(
"--summaries-only: warming summary cache for %s",
repo_filter or "all repos",
)
all_failures["indexing"] = build_txtai_index(
config_path,
articles_config_path,
base_path,
embeddings_path,
dry_run=True, # don't write an index; cache writes happen regardless
category=category,
summary_generator=summary_generator,
repo_filter=repo_filter,
use_cached_ocr_only=use_cached_ocr_only,
)
print_pipeline_summary(all_failures, dry_run)
if env_truthy("NB_STRICT_SUMMARY_WARM"):
indexing_failures = all_failures.get("indexing") or {}
failed_texts = indexing_failures.get("failed_text_files") or []
failed_pdfs = indexing_failures.get("failed_pdf_files") or []
successful_texts = indexing_failures.get("successful_text_files", 0)
successful_pdfs = indexing_failures.get("successful_pdf_files", 0)
if failed_texts or failed_pdfs or (successful_texts + successful_pdfs) == 0:
logger.error(
"Strict summary warm failed: %s text failures, %s PDF failures, %s successful documents.",
len(failed_texts),
len(failed_pdfs),
successful_texts + successful_pdfs,
)
sys.exit(1)
emit_progress(100, stage="done", detail="Summary cache warmed")
return
if articles_config_path and os.path.exists(articles_config_path):
emit_progress(30, stage="articles", detail="Downloading PDF articles")
all_failures["articles"] = download_pdf_articles(
articles_config_path, base_path, dry_run, category, force_update
)
emit_progress(50, stage="articles_done", detail="PDF articles downloaded")
all_failures["indexing"] = build_txtai_index(
config_path,
articles_config_path,
base_path,
embeddings_path,
dry_run,
category,
summary_generator,
repo_filter,
use_cached_ocr_only=use_cached_ocr_only,
)
emit_progress(85, stage="indexing_done", detail="Indexing completed")
if not dirty and not dry_run:
cleanup_raw_repositories(config_path, base_path, category)
if articles_config_path and os.path.exists(articles_config_path):
cleanup_pdf_articles(articles_config_path, base_path, category)
emit_progress(95, stage="cleanup", detail="Cleaning up raw files")
print_pipeline_summary(all_failures, dry_run)
if not dry_run:
indexing_failures = all_failures.get("indexing") or {}
fatal_errors = indexing_failures.get("fatal_errors") or []
if fatal_errors:
for msg in fatal_errors:
logger.error(msg)
sys.exit(1)
failed_texts = indexing_failures.get("failed_text_files") or []
failed_pdfs = indexing_failures.get("failed_pdf_files") or []
if failed_texts or failed_pdfs:
logger.warning(
"Indexing completed with failures. See log output for details on the affected files."
)
emit_progress(100, stage="done", detail="Pipeline finished")
build_pipeline(
config_path=args.config,
articles_config_path=args.articles_config,
base_path=args.base_path,
embeddings_path=args.embeddings_path,
dry_run=args.dry_run,
category=args.category,
force_update=args.force_update,
repo_filter=args.repo,
dirty=args.dirty,
summaries=args.summaries,
summaries_only=args.summaries_only,
use_cached_ocr_only=args.use_cached_ocr_only,
)
"""
This script processes all repositories in the config/repositories.yml file.
Orchestrates the full knowledge base build pipeline (cloning, direct txtai indexing).
"""
import os
import sys
# Allow duplicate OpenMP runtime loading when not explicitly configured. This mirrors
# the CLI behavior so builds don't abort due to multiple libomp copies.
os.environ["KMP_DUPLICATE_LIB_OK"] = "TRUE"
import yaml
import subprocess
from pathlib import Path
import logging
import argparse
import requests
import json
import time
import threading
from typing import Optional, Set, Dict
try:
from dotenv import load_dotenv
except Exception:
load_dotenv = None
from chunky import ChunkPipeline, ChunkerConfig, Document
from nancy_brain.chunking import strip_chunk_suffix
from nancy_brain.pdf_ocr import extract_pdf_markdown, get_pdf_ocr_backend_status
from nancy_brain.summarization import SummaryGenerator
from scripts.script_utils import is_full_commit_sha as _is_full_commit_sha
# Optional imports
try:
from nb4llm import convert_ipynb_to_txt
except ImportError:
convert_ipynb_to_txt = None
# Load .env overrides early so subsequent imports see them
if load_dotenv is not None:
try:
script_root = Path(__file__).resolve().parent.parent
env_path = script_root / "config" / ".env"
if env_path.exists():
load_dotenv(env_path)
except Exception:
pass
# Ensure OpenMP duplicate setting persists after dotenv loads
os.environ["KMP_DUPLICATE_LIB_OK"] = "TRUE"
# PDF exclusion & thresholds
MIN_PDF_BYTES = int(os.environ.get("MIN_PDF_BYTES", "5000")) # skip tiny/image-y PDFs
MIN_PDF_TEXT_CHARS = int(os.environ.get("MIN_PDF_TEXT_CHARS", "500")) # require meaningful text
DEFAULT_EXCLUDE_PDF_SUBSTRINGS = [
"logo/",
"PointSpreadFunctions/",
"PSF_",
"PSFs_",
"PSF-",
"PSF.",
"/graphics/",
"workflow-",
"Glossary.pdf",
"column-mapping.pdf",
]
ENV_EXCLUDES = [e.strip() for e in os.environ.get("PDF_EXCLUDE_SUBSTRINGS", "").split(",") if e.strip()]
EXCLUDE_PDF_SUBSTRINGS = DEFAULT_EXCLUDE_PDF_SUBSTRINGS + ENV_EXCLUDES
README_CANDIDATES = ["README.md", "README.rst", "README.txt"]
DEFAULT_SUMMARIES_ENABLED = os.environ.get("ENABLE_DOC_SUMMARIES", "false").lower() == "true"
SKIP_PDFS = os.environ.get("SKIP_PDF_PROCESSING", "").strip().lower() == "true"
TEXT_EXTENSIONS = {".py", ".md", ".txt", ".rst", ".tex", ".yaml", ".yml", ".json"}
SKIP_DIR_NAMES = {
".git",
".github",
"__pycache__",
"node_modules",
".pytest_cache",
".mypy_cache",
".idea",
".venv",
"env",
"venv",
}
# Minimum content length (stripped) to bother generating a summary.
# Override with NB_MIN_SUMMARY_CHARS environment variable.
MIN_SUMMARY_CHARS = int(os.environ.get("NB_MIN_SUMMARY_CHARS", "200"))
# File extensions for which summary generation is never useful.
# These files are still indexed via their path/doc_id.
SUMMARY_SKIP_EXTENSIONS = {
# Astronomy data
".fits",
".fit",
# Numpy/scipy
".npy",
".npz",
Xet Storage Details
- Size:
- 71 kB
- Xet hash:
- 5720f7d5d1a5367a56b2582eccaf5a1c8e675fd17f1cdb95ee6602a0863b321f
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.