Buckets:

download
raw
35.2 kB
"""Additional tests for scripts/build_knowledge_base.py utility functions."""
import json
import os
import sys
import types
import pytest
import yaml
import subprocess
from pathlib import Path
from unittest.mock import patch, MagicMock, call
import requests
from nancy_brain.pdf_ocr import PDFOCRResult
import scripts.build_knowledge_base as kb_module
from scripts.build_knowledge_base import (
is_excluded_pdf,
load_repo_readme,
collect_repo_files,
emit_progress,
get_file_type_category,
process_pdf_with_ocr,
download_pdf_articles,
clone_repositories,
)
# ---------------------------------------------------------------------------
# is_excluded_pdf
# ---------------------------------------------------------------------------
def test_is_excluded_pdf_logo():
assert is_excluded_pdf("repo/logo/icon.pdf") is True
def test_is_excluded_pdf_psf():
assert is_excluded_pdf("/data/PSF_model.pdf") is True
def test_is_excluded_pdf_glossary():
assert is_excluded_pdf("docs/Glossary.pdf") is True
def test_is_excluded_pdf_normal():
assert is_excluded_pdf("/science/repo/paper.pdf") is False
def test_is_excluded_pdf_graphics():
assert is_excluded_pdf("/repo/graphics/figure.pdf") is True
# ---------------------------------------------------------------------------
# load_repo_readme
# ---------------------------------------------------------------------------
def test_load_repo_readme_finds_readme_md(tmp_path):
readme = tmp_path / "README.md"
readme.write_text("# Hello\nThis is a test.", encoding="utf-8")
result = load_repo_readme(tmp_path)
assert result is not None
assert "Hello" in result["content"]
assert result["path"] == "README.md"
def test_load_repo_readme_finds_readme_rst(tmp_path):
readme = tmp_path / "README.rst"
readme.write_text("Title\n=====\nSome text.", encoding="utf-8")
result = load_repo_readme(tmp_path)
assert result is not None
assert result["path"] == "README.rst"
def test_load_repo_readme_empty_readme(tmp_path):
readme = tmp_path / "README.md"
readme.write_text(" \n ", encoding="utf-8")
result = load_repo_readme(tmp_path)
assert result is None
def test_load_repo_readme_no_readme(tmp_path):
result = load_repo_readme(tmp_path)
assert result is None
# ---------------------------------------------------------------------------
# collect_repo_files
# ---------------------------------------------------------------------------
def test_collect_repo_files_text_files(tmp_path):
(tmp_path / "main.py").write_text("print('hi')")
(tmp_path / "readme.md").write_text("# README")
(tmp_path / "config.yml").write_text("key: value")
text_files, pdf_files = collect_repo_files(tmp_path)
assert len(text_files) == 3
assert len(pdf_files) == 0
def test_collect_repo_files_with_pdf(tmp_path, monkeypatch):
monkeypatch.setattr(kb_module, "SKIP_PDFS", False)
(tmp_path / "paper.pdf").write_bytes(b"%PDF-1.4 content")
text_files, pdf_files = collect_repo_files(tmp_path)
assert len(pdf_files) == 1
def test_collect_repo_files_skips_git_dir(tmp_path):
git_dir = tmp_path / ".git"
git_dir.mkdir()
(git_dir / "config").write_text("repo config")
text_files, pdf_files = collect_repo_files(tmp_path)
assert not any(".git" in str(f) for f in text_files)
def test_collect_repo_files_nb_txt(tmp_path):
(tmp_path / "notebook.nb.txt").write_text("notebook content")
text_files, _ = collect_repo_files(tmp_path)
assert any("nb.txt" in str(f) for f in text_files)
# ---------------------------------------------------------------------------
# emit_progress
# ---------------------------------------------------------------------------
def test_emit_progress_outputs_json(capsys):
emit_progress(50, stage="cloning", detail="repo-x")
captured = capsys.readouterr()
assert "PROGRESS_JSON:" in captured.out
import json
data = json.loads(captured.out.split("PROGRESS_JSON: ")[1])
assert data["percent"] == 50
assert data["stage"] == "cloning"
assert data["detail"] == "repo-x"
def test_emit_progress_no_raise_on_exception():
# Should never raise
emit_progress("not-an-int", stage=None, detail=None)
# ---------------------------------------------------------------------------
# get_file_type_category
# ---------------------------------------------------------------------------
def test_get_file_type_category_python():
assert get_file_type_category("repo/script.py") == "code"
def test_get_file_type_category_js():
assert get_file_type_category("repo/app.js") == "code"
def test_get_file_type_category_markdown():
assert get_file_type_category("docs/guide.md") == "mixed"
def test_get_file_type_category_yaml():
assert get_file_type_category("config.yaml") == "mixed"
def test_get_file_type_category_notebook():
assert get_file_type_category("notebooks/analysis.nb.txt") == "mixed"
def test_get_file_type_category_rst():
assert get_file_type_category("docs/api.rst") == "mixed"
def test_get_file_type_category_other():
assert get_file_type_category("data/results.csv") == "docs"
def test_get_file_type_category_cpp():
assert get_file_type_category("src/main.cpp") == "code"
# ---------------------------------------------------------------------------
# process_pdf_with_ocr
# ---------------------------------------------------------------------------
def test_process_pdf_with_ocr_unavailable(tmp_path, monkeypatch):
pdf_path = tmp_path / "test.pdf"
pdf_path.write_bytes(b"fake pdf content")
mocked_result = PDFOCRResult(
markdown=None,
backend="none",
cached=False,
cache_key="abc123",
status="needs_ocr",
needs_ocr=True,
deferred=True,
warning="CUDA unavailable",
)
with patch.object(kb_module, "extract_pdf_markdown", return_value=mocked_result):
content, ocr_result, success = process_pdf_with_ocr(pdf_path)
assert content is None
assert isinstance(success, bool)
assert ocr_result.warning == "CUDA unavailable"
assert ocr_result.needs_ocr is True
def test_process_pdf_with_ocr_success(tmp_path, monkeypatch):
pdf_path = tmp_path / "test.pdf"
pdf_path.write_bytes(b"fake pdf content " * 50)
cache_dir = tmp_path / "cache"
mocked_result = PDFOCRResult(
markdown="extracted markdown " * 50,
backend="deepseek",
cached=False,
cache_key="abc123",
model="deepseek-ai/DeepSeek-OCR",
page_count=2,
)
with patch.object(kb_module, "extract_pdf_markdown", return_value=mocked_result) as mock_extract:
content, ocr_result, success = process_pdf_with_ocr(pdf_path, cache_dir=cache_dir)
assert success is True
assert content is not None
assert ocr_result.backend == "deepseek"
mock_extract.assert_called_once_with(pdf_path, cache_dir=cache_dir, preferred_backend=None)
def test_process_pdf_with_ocr_cache_only_uses_skip_backend(tmp_path):
pdf_path = tmp_path / "test.pdf"
pdf_path.write_bytes(b"fake pdf content " * 50)
mocked_result = PDFOCRResult(
markdown="cached markdown " * 50,
backend="deepseek",
cached=True,
cache_key="abc123",
status="cached",
)
with patch.object(kb_module, "extract_pdf_markdown", return_value=mocked_result) as mock_extract:
content, ocr_result, success = process_pdf_with_ocr(pdf_path, use_cached_ocr_only=True)
assert success is True
assert content.startswith("cached markdown")
assert ocr_result.cached is True
mock_extract.assert_called_once_with(pdf_path, cache_dir=None, preferred_backend="skip")
def test_process_pdf_with_ocr_content_too_short(tmp_path, monkeypatch):
pdf_path = tmp_path / "tiny.pdf"
pdf_path.write_bytes(b"fake pdf")
monkeypatch.setattr(kb_module, "MIN_PDF_TEXT_CHARS", 10000) # Very high threshold
mocked_result = PDFOCRResult(
markdown="short",
backend="deepseek",
cached=False,
cache_key="abc123",
)
with patch.object(kb_module, "extract_pdf_markdown", return_value=mocked_result):
content, _, success = process_pdf_with_ocr(pdf_path)
assert success is False
# ---------------------------------------------------------------------------
# download_pdf_articles
# ---------------------------------------------------------------------------
def _make_requests_response(content=b"A" * 10000, content_type="application/pdf", status_code=200):
mock = MagicMock()
mock.status_code = status_code
mock.content = content
mock.headers = {"Content-Type": content_type}
mock.raise_for_status = MagicMock()
return mock
def _write_articles_config(path: Path, config: dict) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
with open(path, "w", encoding="utf-8") as f:
yaml.safe_dump(config, f)
def test_download_pdf_success(tmp_path):
config_path = tmp_path / "articles.yml"
_write_articles_config(config_path, {"papers": [{"name": "paper1", "url": "https://example.com/paper1.pdf"}]})
mock_session = MagicMock()
mock_session.get.return_value = _make_requests_response()
mock_session.max_redirects = 15
with patch("scripts.build_knowledge_base.requests.Session", return_value=mock_session):
result = download_pdf_articles(str(config_path), base_path=str(tmp_path / "raw"))
assert "paper1" in str(result["successful_downloads"])
def test_download_pdf_already_exists(tmp_path):
config_path = tmp_path / "articles.yml"
_write_articles_config(config_path, {"papers": [{"name": "existing", "url": "https://example.com/existing.pdf"}]})
# Create the file
dest = tmp_path / "raw" / "papers" / "existing.pdf"
dest.parent.mkdir(parents=True, exist_ok=True)
dest.write_bytes(b"existing content")
with patch("scripts.build_knowledge_base.requests.Session"):
result = download_pdf_articles(str(config_path), base_path=str(tmp_path / "raw"))
assert "papers/existing" in result["skipped_existing"]
def test_download_pdf_dry_run(tmp_path):
config_path = tmp_path / "articles.yml"
_write_articles_config(config_path, {"papers": [{"name": "test", "url": "https://example.com/test.pdf"}]})
mock_session = MagicMock()
with patch("scripts.build_knowledge_base.requests.Session", return_value=mock_session):
result = download_pdf_articles(str(config_path), base_path=str(tmp_path / "raw"), dry_run=True)
mock_session.get.assert_not_called()
assert not (tmp_path / "raw" / "papers" / "test.pdf").exists()
def test_download_pdf_failure(tmp_path):
config_path = tmp_path / "articles.yml"
_write_articles_config(config_path, {"papers": [{"name": "bad", "url": "https://example.com/bad.pdf"}]})
mock_session = MagicMock()
mock_session.get.side_effect = requests.ConnectionError("connection refused")
with patch("scripts.build_knowledge_base.requests.Session", return_value=mock_session):
result = download_pdf_articles(str(config_path), base_path=str(tmp_path / "raw"))
assert any("bad" in entry for entry in result["failed_downloads"])
def test_download_pdf_html_response(tmp_path):
"""PDF download that returns HTML (anti-scraping) should fail."""
config_path = tmp_path / "articles.yml"
_write_articles_config(config_path, {"papers": [{"name": "html-paper", "url": "https://example.com/paper.pdf"}]})
mock_session = MagicMock()
mock_session.get.return_value = _make_requests_response(
content=b"<html>Access denied</html>",
content_type="text/html",
)
with patch("scripts.build_knowledge_base.requests.Session", return_value=mock_session):
result = download_pdf_articles(str(config_path), base_path=str(tmp_path / "raw"))
assert any("html-paper" in entry for entry in result["failed_downloads"])
def test_download_pdf_with_category_filter(tmp_path):
config_path = tmp_path / "articles.yml"
_write_articles_config(
config_path,
{
"papers": [{"name": "p1", "url": "https://example.com/p1.pdf"}],
"other": [{"name": "p2", "url": "https://example.com/p2.pdf"}],
},
)
mock_session = MagicMock()
mock_session.get.return_value = _make_requests_response()
with patch("scripts.build_knowledge_base.requests.Session", return_value=mock_session):
result = download_pdf_articles(str(config_path), base_path=str(tmp_path / "raw"), category="papers")
# Only paper from "papers" category should be processed
assert mock_session.get.call_count == 1
# ---------------------------------------------------------------------------
# clone_repositories - extended scenarios
# ---------------------------------------------------------------------------
def _write_repos_config(path: Path, config: dict) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
with open(path, "w", encoding="utf-8") as f:
yaml.safe_dump(config, f)
def test_clone_repositories_skips_existing(tmp_path):
config_path = tmp_path / "repos.yml"
_write_repos_config(config_path, {"science": [{"name": "existing", "url": "https://github.com/org/existing.git"}]})
dest = tmp_path / "raw" / "science" / "existing"
dest.mkdir(parents=True, exist_ok=True)
with patch("scripts.build_knowledge_base.subprocess.run"):
result = clone_repositories(str(config_path), base_path=str(tmp_path / "raw"))
assert "science/existing" in result["skipped_existing"]
def test_clone_repositories_dry_run(tmp_path):
config_path = tmp_path / "repos.yml"
_write_repos_config(config_path, {"science": [{"name": "new-repo", "url": "https://github.com/org/new-repo.git"}]})
with patch("scripts.build_knowledge_base.subprocess.run") as mock_run:
result = clone_repositories(str(config_path), base_path=str(tmp_path / "raw"), dry_run=True)
mock_run.assert_not_called()
def test_clone_repositories_force_update_existing(tmp_path):
config_path = tmp_path / "repos.yml"
_write_repos_config(config_path, {"science": [{"name": "repo", "url": "https://github.com/org/repo.git"}]})
dest = tmp_path / "raw" / "science" / "repo"
dest.mkdir(parents=True, exist_ok=True)
with patch("scripts.build_knowledge_base.subprocess.run") as mock_run:
mock_run.return_value = MagicMock(returncode=0, stdout="main\n")
result = clone_repositories(str(config_path), base_path=str(tmp_path / "raw"), force_update=True)
assert "science/repo" in result["successful_updates"]
def test_clone_repositories_force_update_with_ref(tmp_path):
config_path = tmp_path / "repos.yml"
_write_repos_config(
config_path, {"science": [{"name": "repo", "url": "https://github.com/org/repo.git", "ref": "v1.0"}]}
)
dest = tmp_path / "raw" / "science" / "repo"
dest.mkdir(parents=True, exist_ok=True)
with patch("scripts.build_knowledge_base.subprocess.run") as mock_run:
mock_run.return_value = MagicMock(returncode=0, stdout="main\n")
result = clone_repositories(str(config_path), base_path=str(tmp_path / "raw"), force_update=True)
assert "science/repo" in result["successful_updates"]
def test_clone_repositories_force_update_with_sha_ref(tmp_path):
sha = "a" * 40
config_path = tmp_path / "repos.yml"
_write_repos_config(
config_path, {"science": [{"name": "repo", "url": "https://github.com/org/repo.git", "ref": sha}]}
)
dest = tmp_path / "raw" / "science" / "repo"
dest.mkdir(parents=True, exist_ok=True)
with patch("scripts.build_knowledge_base.subprocess.run") as mock_run:
mock_run.return_value = MagicMock(returncode=0, stdout="main\n")
result = clone_repositories(str(config_path), base_path=str(tmp_path / "raw"), force_update=True)
assert "science/repo" in result["successful_updates"]
def test_clone_repositories_update_failure(tmp_path):
config_path = tmp_path / "repos.yml"
_write_repos_config(config_path, {"science": [{"name": "repo", "url": "https://github.com/org/repo.git"}]})
dest = tmp_path / "raw" / "science" / "repo"
dest.mkdir(parents=True, exist_ok=True)
with patch("scripts.build_knowledge_base.subprocess.run") as mock_run:
mock_run.side_effect = subprocess.CalledProcessError(1, ["git"], stderr="remote error")
result = clone_repositories(str(config_path), base_path=str(tmp_path / "raw"), force_update=True)
assert any("repo" in entry for entry in result["failed_updates"])
def test_clone_repositories_clone_with_ref(tmp_path):
config_path = tmp_path / "repos.yml"
_write_repos_config(
config_path, {"science": [{"name": "ref-repo", "url": "https://github.com/org/repo.git", "ref": "v2.0"}]}
)
with patch("scripts.build_knowledge_base.subprocess.run") as mock_run:
mock_run.return_value = MagicMock(returncode=0)
result = clone_repositories(str(config_path), base_path=str(tmp_path / "raw"))
# Should have cloned with --branch v2.0
clone_calls = [c.args[0] for c in mock_run.call_args_list if c.args and c.args[0][:2] == ["git", "clone"]]
assert any("--branch" in clc and "v2.0" in clc for clc in clone_calls)
def test_clone_repositories_clone_failure(tmp_path):
config_path = tmp_path / "repos.yml"
_write_repos_config(config_path, {"science": [{"name": "bad", "url": "https://github.com/org/bad.git"}]})
with patch("scripts.build_knowledge_base.subprocess.run") as mock_run:
mock_run.side_effect = subprocess.CalledProcessError(128, ["git"], stderr="not found")
result = clone_repositories(str(config_path), base_path=str(tmp_path / "raw"))
assert any("bad" in entry for entry in result["failed_clones"])
def test_clone_repositories_dry_run_existing(tmp_path):
config_path = tmp_path / "repos.yml"
_write_repos_config(config_path, {"science": [{"name": "repo", "url": "https://github.com/org/repo.git"}]})
dest = tmp_path / "raw" / "science" / "repo"
dest.mkdir(parents=True, exist_ok=True)
with patch("scripts.build_knowledge_base.subprocess.run") as mock_run:
result = clone_repositories(str(config_path), base_path=str(tmp_path / "raw"), dry_run=True, force_update=True)
mock_run.assert_not_called()
def test_clone_repositories_with_sha_ref_clone(tmp_path):
sha = "b" * 40
config_path = tmp_path / "repos.yml"
_write_repos_config(
config_path, {"science": [{"name": "sha-repo", "url": "https://github.com/org/repo.git", "ref": sha}]}
)
with patch("scripts.build_knowledge_base.subprocess.run") as mock_run:
mock_run.return_value = MagicMock(returncode=0)
result = clone_repositories(str(config_path), base_path=str(tmp_path / "raw"))
# Should have cloned without --branch and then checked out the SHA
calls = [c.args[0] for c in mock_run.call_args_list]
clone_calls = [c for c in calls if c[:2] == ["git", "clone"]]
checkout_calls = [c for c in calls if c[:2] == ["git", "checkout"]]
assert len(clone_calls) == 1
assert "--branch" not in clone_calls[0]
assert any(sha in c for c in checkout_calls)
# ---------------------------------------------------------------------------
# cleanup_pdf_articles
# ---------------------------------------------------------------------------
from scripts.build_knowledge_base import cleanup_pdf_articles, cleanup_raw_repositories
def test_cleanup_pdf_articles_removes_files(tmp_path):
config_path = tmp_path / "articles.yml"
_write_articles_config(config_path, {"papers": [{"name": "paper1", "url": "https://example.com/paper1.pdf"}]})
pdf_file = tmp_path / "raw" / "papers" / "paper1.pdf"
pdf_file.parent.mkdir(parents=True, exist_ok=True)
pdf_file.write_bytes(b"fake pdf")
cleanup_pdf_articles(str(config_path), base_path=str(tmp_path / "raw"))
assert not pdf_file.exists()
def test_cleanup_pdf_articles_handles_missing_file(tmp_path):
config_path = tmp_path / "articles.yml"
_write_articles_config(config_path, {"papers": [{"name": "nonexistent", "url": "https://example.com/p.pdf"}]})
# Should not raise even if file doesn't exist
cleanup_pdf_articles(str(config_path), base_path=str(tmp_path / "raw"))
def test_cleanup_pdf_articles_removes_empty_dirs(tmp_path):
config_path = tmp_path / "articles.yml"
_write_articles_config(config_path, {"papers": [{"name": "paper1", "url": "https://example.com/p.pdf"}]})
# Create empty category dir
empty_dir = tmp_path / "raw" / "papers"
empty_dir.mkdir(parents=True, exist_ok=True)
pdf = empty_dir / "paper1.pdf"
pdf.write_bytes(b"fake")
cleanup_pdf_articles(str(config_path), base_path=str(tmp_path / "raw"))
# The empty dir should be removed after pdf is deleted
assert not empty_dir.exists()
def test_cleanup_pdf_articles_with_category_filter(tmp_path):
config_path = tmp_path / "articles.yml"
_write_articles_config(
config_path,
{
"papers": [{"name": "p1", "url": "https://example.com/p1.pdf"}],
"other": [{"name": "p2", "url": "https://example.com/p2.pdf"}],
},
)
pdf1 = tmp_path / "raw" / "papers" / "p1.pdf"
pdf2 = tmp_path / "raw" / "other" / "p2.pdf"
pdf1.parent.mkdir(parents=True, exist_ok=True)
pdf2.parent.mkdir(parents=True, exist_ok=True)
pdf1.write_bytes(b"pdf1")
pdf2.write_bytes(b"pdf2")
cleanup_pdf_articles(str(config_path), base_path=str(tmp_path / "raw"), category="papers")
assert not pdf1.exists()
assert pdf2.exists()
# ---------------------------------------------------------------------------
# cleanup_raw_repositories
# ---------------------------------------------------------------------------
def test_cleanup_raw_repositories_removes_dirs(tmp_path):
config_path = tmp_path / "repos.yml"
_write_repos_config(config_path, {"science": [{"name": "myrepo", "url": "https://github.com/org/repo.git"}]})
repo_dir = tmp_path / "raw" / "science" / "myrepo"
repo_dir.mkdir(parents=True, exist_ok=True)
(repo_dir / "file.py").write_text("code")
cleanup_raw_repositories(str(config_path), base_path=str(tmp_path / "raw"))
assert not repo_dir.exists()
def test_cleanup_raw_repositories_handles_missing(tmp_path):
config_path = tmp_path / "repos.yml"
_write_repos_config(config_path, {"science": [{"name": "ghost", "url": "https://github.com/org/ghost.git"}]})
# Should not raise even if directory doesn't exist
cleanup_raw_repositories(str(config_path), base_path=str(tmp_path / "raw"))
def test_cleanup_raw_repositories_removes_empty_cat_dirs(tmp_path):
config_path = tmp_path / "repos.yml"
_write_repos_config(config_path, {"science": [{"name": "repo", "url": "https://github.com/org/repo.git"}]})
repo_dir = tmp_path / "raw" / "science" / "repo"
repo_dir.mkdir(parents=True, exist_ok=True)
(repo_dir / "main.py").write_text("code")
cleanup_raw_repositories(str(config_path), base_path=str(tmp_path / "raw"))
# Category dir should be removed too since it's now empty
assert not (tmp_path / "raw" / "science").exists()
def test_cleanup_raw_repositories_with_category_filter(tmp_path):
config_path = tmp_path / "repos.yml"
_write_repos_config(
config_path,
{
"science": [{"name": "keep_science", "url": "https://github.com/org/r.git"}],
"tools": [{"name": "del_tools", "url": "https://github.com/org/t.git"}],
},
)
science_dir = tmp_path / "raw" / "science" / "keep_science"
tools_dir = tmp_path / "raw" / "tools" / "del_tools"
science_dir.mkdir(parents=True, exist_ok=True)
tools_dir.mkdir(parents=True, exist_ok=True)
(science_dir / "f.py").write_text("x")
(tools_dir / "g.py").write_text("y")
cleanup_raw_repositories(str(config_path), base_path=str(tmp_path / "raw"), category="tools")
assert science_dir.exists()
assert not tools_dir.exists()
# ---------------------------------------------------------------------------
# Additional build_txtai_index tests
# ---------------------------------------------------------------------------
from scripts.build_knowledge_base import build_txtai_index
class DummyEmbeddings2:
instances = []
def __init__(self, *args, **kwargs):
self.indexed = []
DummyEmbeddings2.instances.append(self)
def index(self, docs):
self.indexed.extend(list(docs))
def save(self, path):
from pathlib import Path
Path(path).mkdir(parents=True, exist_ok=True)
def search(self, query, limit):
return []
def _install_embeddings(monkeypatch):
DummyEmbeddings2.instances.clear()
txtai_module = types.ModuleType("txtai")
emb_module = types.ModuleType("txtai.embeddings")
emb_module.Embeddings = DummyEmbeddings2
monkeypatch.setitem(sys.modules, "txtai", txtai_module)
monkeypatch.setitem(sys.modules, "txtai.embeddings", emb_module)
def test_build_txtai_index_skips_missing_repo(monkeypatch, tmp_path):
"""Repos whose directories don't exist are reported as skipped."""
_install_embeddings(monkeypatch)
monkeypatch.setenv("USE_DUAL_EMBEDDING", "false")
config_path = tmp_path / "repositories.yml"
config_path.parent.mkdir(parents=True, exist_ok=True)
config_path.write_text(
"science:\n - name: ghost-repo\n url: https://example.com/ghost.git\n",
encoding="utf-8",
)
failures = build_txtai_index(
str(config_path),
base_path=str(tmp_path / "raw"),
embeddings_path=str(tmp_path / "embeddings"),
)
assert "science/ghost-repo" in failures["skipped_repositories"]
def test_build_txtai_index_validation_error(monkeypatch, tmp_path):
"""Invalid config triggers validation error."""
_install_embeddings(monkeypatch)
monkeypatch.setenv("USE_DUAL_EMBEDDING", "false")
config_path = tmp_path / "repositories.yml"
config_path.parent.mkdir(parents=True, exist_ok=True)
# Missing 'url' key triggers validation error
config_path.write_text(
"science:\n - name: bad\n",
encoding="utf-8",
)
failures = build_txtai_index(
str(config_path),
base_path=str(tmp_path / "raw"),
embeddings_path=str(tmp_path / "embeddings"),
)
assert "validation_errors" in failures
def test_build_txtai_index_dry_run(monkeypatch, tmp_path):
"""In dry_run mode, no index should be built."""
_install_embeddings(monkeypatch)
monkeypatch.setenv("USE_DUAL_EMBEDDING", "false")
config_path = tmp_path / "repositories.yml"
config_path.parent.mkdir(parents=True, exist_ok=True)
config_path.write_text(
"science:\n - name: repo1\n url: https://github.com/org/repo.git\n",
encoding="utf-8",
)
repo_dir = tmp_path / "raw" / "science" / "repo1"
repo_dir.mkdir(parents=True, exist_ok=True)
(repo_dir / "main.py").write_text("def main(): pass\n" * 50, encoding="utf-8")
failures = build_txtai_index(
str(config_path),
base_path=str(tmp_path / "raw"),
embeddings_path=str(tmp_path / "embeddings"),
dry_run=True,
)
# No embeddings should be saved when dry_run
index_path = tmp_path / "embeddings" / "index"
assert not index_path.exists() or failures is not None
def test_build_txtai_index_with_pdf_file(monkeypatch, tmp_path):
"""PDF files are processed even when OCR produces no markdown."""
_install_embeddings(monkeypatch)
monkeypatch.setenv("USE_DUAL_EMBEDDING", "false")
monkeypatch.setenv("SKIP_PDF_PROCESSING", "")
monkeypatch.setattr(kb_module, "SKIP_PDFS", False)
config_path = tmp_path / "repositories.yml"
config_path.parent.mkdir(parents=True, exist_ok=True)
config_path.write_text(
"science:\n - name: pdf-repo\n url: https://example.com/repo.git\n",
encoding="utf-8",
)
repo_dir = tmp_path / "raw" / "science" / "pdf-repo"
repo_dir.mkdir(parents=True, exist_ok=True)
# Create a real PDF-sized file
pdf_path = repo_dir / "report.pdf"
pdf_path.write_bytes(b"%PDF-1.4 " + b"x" * 10000)
mocked_result = PDFOCRResult(
markdown=None,
backend="none",
cached=False,
cache_key="abc123",
warning="OCR unavailable",
)
with patch.object(kb_module, "extract_pdf_markdown", return_value=mocked_result):
failures = build_txtai_index(
str(config_path),
base_path=str(tmp_path / "raw"),
embeddings_path=str(tmp_path / "embeddings"),
)
# Either failed to extract or succeeded
assert isinstance(failures, dict)
def test_build_txtai_index_treats_missing_ocr_as_failure_by_default(monkeypatch, tmp_path):
"""Default builds should fail PDF indexing when OCR is unavailable."""
_install_embeddings(monkeypatch)
monkeypatch.setenv("USE_DUAL_EMBEDDING", "false")
monkeypatch.setattr(kb_module, "SKIP_PDFS", False)
config_path = tmp_path / "repositories.yml"
config_path.parent.mkdir(parents=True, exist_ok=True)
config_path.write_text(
"science:\n - name: pdf-repo\n url: https://example.com/repo.git\n",
encoding="utf-8",
)
repo_dir = tmp_path / "raw" / "science" / "pdf-repo"
repo_dir.mkdir(parents=True, exist_ok=True)
pdf_path = repo_dir / "report.pdf"
pdf_path.write_bytes(b"%PDF-1.4 " + b"x" * 10000)
mocked_result = PDFOCRResult(
markdown=None,
backend="none",
cached=False,
cache_key="abc123",
status="needs_ocr",
needs_ocr=True,
deferred=True,
warning="OCR unavailable",
)
with (
patch.object(kb_module, "collect_repo_files", return_value=([], [pdf_path])),
patch.object(kb_module, "process_pdf_with_ocr", return_value=(None, mocked_result, False)),
):
failures = build_txtai_index(
str(config_path),
base_path=str(tmp_path / "raw"),
embeddings_path=str(tmp_path / "embeddings"),
)
assert failures["failed_pdf_files"]
assert not failures["deferred_pdf_files"]
def test_build_txtai_index_with_pdf_success(monkeypatch, tmp_path):
"""PDF files with successful extraction get indexed."""
_install_embeddings(monkeypatch)
monkeypatch.setenv("USE_DUAL_EMBEDDING", "false")
monkeypatch.setattr(kb_module, "SKIP_PDFS", False)
config_path = tmp_path / "repositories.yml"
config_path.parent.mkdir(parents=True, exist_ok=True)
config_path.write_text(
"science:\n - name: pdf-ok-repo\n url: https://example.com/repo.git\n",
encoding="utf-8",
)
repo_dir = tmp_path / "raw" / "science" / "pdf-ok-repo"
repo_dir.mkdir(parents=True, exist_ok=True)
pdf_path = repo_dir / "paper.pdf"
pdf_path.write_bytes(b"%PDF-1.4 " + b"x" * 10000)
mocked_result = PDFOCRResult(
markdown="# Extracted PDF\n\n" + ("This is extracted PDF content. " * 100),
backend="deepseek",
cached=False,
cache_key="abc123",
model="deepseek-ai/DeepSeek-OCR",
page_count=1,
)
with patch.object(kb_module, "extract_pdf_markdown", return_value=mocked_result):
failures = build_txtai_index(
str(config_path),
base_path=str(tmp_path / "raw"),
embeddings_path=str(tmp_path / "embeddings"),
)
assert failures["successful_pdf_files"] >= 1
indexed_docs = DummyEmbeddings2.instances[0].indexed
pdf_chunks = [
doc for doc in indexed_docs if json.loads(doc[2]).get("source_document") == "science/pdf-ok-repo/paper.pdf"
]
assert pdf_chunks
pdf_metadata = [json.loads(doc[2]) for doc in pdf_chunks]
assert all(meta.get("ocr_backend") == "deepseek" for meta in pdf_metadata)
def test_build_txtai_index_text_file_exception(monkeypatch, tmp_path):
"""Exception in text file reading is caught and recorded."""
_install_embeddings(monkeypatch)
monkeypatch.setenv("USE_DUAL_EMBEDDING", "false")
config_path = tmp_path / "repositories.yml"
config_path.parent.mkdir(parents=True, exist_ok=True)
config_path.write_text(
"science:\n - name: repo\n url: https://github.com/org/repo.git\n",
encoding="utf-8",
)
repo_dir = tmp_path / "raw" / "science" / "repo"
repo_dir.mkdir(parents=True, exist_ok=True)
# Create a .py file
py_file = repo_dir / "bad.py"
py_file.write_text("content here " * 50, encoding="utf-8")
original_open = open
call_count = 0
def mock_open(path, *args, **kwargs):
nonlocal call_count
call_count += 1
if str(path).endswith("bad.py") and call_count > 2:
raise IOError("file read error")
return original_open(path, *args, **kwargs)
with patch("builtins.open", side_effect=mock_open):
failures = build_txtai_index(
str(config_path),
base_path=str(tmp_path / "raw"),
embeddings_path=str(tmp_path / "embeddings"),
)
# Should not crash; either succeeded or failed gracefully
assert isinstance(failures, dict)
def test_build_txtai_index_with_articles_config(monkeypatch, tmp_path):
"""Articles config is loaded when provided."""
_install_embeddings(monkeypatch)
monkeypatch.setenv("USE_DUAL_EMBEDDING", "false")
repos_path = tmp_path / "repositories.yml"
repos_path.parent.mkdir(parents=True, exist_ok=True)
repos_path.write_text(
"science:\n - name: repo1\n url: https://example.com/repo.git\n",
encoding="utf-8",
)
articles_path = tmp_path / "articles.yml"
articles_path.write_text(
"papers:\n - name: paper1\n url: https://example.com/p.pdf\n description: Paper 1\n",
encoding="utf-8",
)
# Don't create the repo or article dirs -> both should be skipped
failures = build_txtai_index(
str(repos_path),
articles_config_path=str(articles_path),
base_path=str(tmp_path / "raw"),
embeddings_path=str(tmp_path / "embeddings"),
)
assert "papers/paper1" in failures["skipped_articles"]
def test_build_txtai_index_articles_validation_error(monkeypatch, tmp_path):
"""Invalid articles config triggers validation error."""
_install_embeddings(monkeypatch)
monkeypatch.setenv("USE_DUAL_EMBEDDING", "false")
repos_path = tmp_path / "repositories.yml"
repos_path.parent.mkdir(parents=True, exist_ok=True)
repos_path.write_text(
"science:\n - name: repo1\n url: https://github.com/org/repo.git\n",
encoding="utf-8",
)
articles_path = tmp_path / "articles.yml"
articles_path.write_text(
"papers:\n - name: bad_article\n", # missing url
encoding="utf-8",
)
failures = build_txtai_index(
str(repos_path),
articles_config_path=str(articles_path),
base_path=str(tmp_path / "raw"),
embeddings_path=str(tmp_path / "embeddings"),
)
assert "validation_errors" in failures

Xet Storage Details

Size:
35.2 kB
·
Xet hash:
2995d756b5024cb2dff6bb9d0c299b34c0693105b8b8c811f6c8115b8196a5c2

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.