Spaces:
Sleeping
Sleeping
Islam Mamedov commited on
Commit ·
98aa8c9
1
Parent(s): 3df8a15
Day 2: AST-aware chunkers for code, docs, issues
Browse files- data/chunks.jsonl +0 -0
- src/__pycache__/chunk.cpython-311.pyc +0 -0
- src/chunk.py +248 -0
- tests/__pycache__/test_chunk.cpython-311-pytest-9.1.1.pyc +0 -0
- tests/test_chunk.py +114 -0
data/chunks.jsonl
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
src/__pycache__/chunk.cpython-311.pyc
ADDED
|
Binary file (14.2 kB). View file
|
|
|
src/chunk.py
ADDED
|
@@ -0,0 +1,248 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Chunk ingested content into retrieval units.
|
| 2 |
+
|
| 3 |
+
Reads data/manifest.json + files produced by ingest.py.
|
| 4 |
+
Writes data/chunks.jsonl — one JSON chunk per line.
|
| 5 |
+
|
| 6 |
+
Chunk schema (same for all source types):
|
| 7 |
+
id stable unique id
|
| 8 |
+
text what gets embedded (includes a context header)
|
| 9 |
+
source_type code | doc | issue
|
| 10 |
+
path file path in the repo (or issues/N.json)
|
| 11 |
+
symbol function/class name, doc heading, or issue title
|
| 12 |
+
url link to the exact GitHub location
|
| 13 |
+
start_line / end_line (code and docs only)
|
| 14 |
+
|
| 15 |
+
Usage:
|
| 16 |
+
python src/chunk.py
|
| 17 |
+
"""
|
| 18 |
+
|
| 19 |
+
import json
|
| 20 |
+
import subprocess
|
| 21 |
+
from pathlib import Path
|
| 22 |
+
|
| 23 |
+
import tree_sitter_python as tspython
|
| 24 |
+
from tree_sitter import Language, Parser
|
| 25 |
+
|
| 26 |
+
DATA_DIR = Path("data")
|
| 27 |
+
REPO_DIR = DATA_DIR / "repo"
|
| 28 |
+
MAX_CLASS_LINES = 150 # classes bigger than this get split into methods
|
| 29 |
+
MIN_CHUNK_CHARS = 50 # drop trivial fragments
|
| 30 |
+
MAX_ISSUE_BODY = 3000
|
| 31 |
+
MAX_COMMENT = 1000
|
| 32 |
+
|
| 33 |
+
PY_LANGUAGE = Language(tspython.language())
|
| 34 |
+
PARSER = Parser(PY_LANGUAGE)
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
# ---------------------------------------------------------------- helpers
|
| 38 |
+
|
| 39 |
+
def default_branch() -> str:
|
| 40 |
+
out = subprocess.run(["git", "-C", str(REPO_DIR), "rev-parse",
|
| 41 |
+
"--abbrev-ref", "HEAD"],
|
| 42 |
+
capture_output=True, text=True)
|
| 43 |
+
return out.stdout.strip() or "master"
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
def github_url(repo: str, branch: str, path: str,
|
| 47 |
+
start: int | None = None, end: int | None = None) -> str:
|
| 48 |
+
url = f"https://github.com/{repo}/blob/{branch}/{path}"
|
| 49 |
+
if start is not None:
|
| 50 |
+
url += f"#L{start}-L{end}"
|
| 51 |
+
return url
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
# ---------------------------------------------------------------- code
|
| 55 |
+
|
| 56 |
+
def chunk_code_file(source: bytes, rel_path: str, repo: str,
|
| 57 |
+
branch: str) -> list[dict]:
|
| 58 |
+
"""AST-aware chunking: one chunk per top-level function/class.
|
| 59 |
+
|
| 60 |
+
Large classes are split into per-method chunks. Module-level leftovers
|
| 61 |
+
(imports, constants) become one '<module>' chunk.
|
| 62 |
+
"""
|
| 63 |
+
tree = PARSER.parse(source)
|
| 64 |
+
text_lines = source.decode(errors="replace").splitlines()
|
| 65 |
+
chunks: list[dict] = []
|
| 66 |
+
covered: set[int] = set() # 1-based line numbers already in a chunk
|
| 67 |
+
|
| 68 |
+
def node_text(node) -> str:
|
| 69 |
+
return source[node.start_byte:node.end_byte].decode(errors="replace")
|
| 70 |
+
|
| 71 |
+
def emit(node, symbol: str, context: str = "") -> None:
|
| 72 |
+
start, end = node.start_point[0] + 1, node.end_point[0] + 1
|
| 73 |
+
header = f"# File: {rel_path}"
|
| 74 |
+
if context:
|
| 75 |
+
header += f" | {context}"
|
| 76 |
+
header += f" | Symbol: {symbol}\n"
|
| 77 |
+
body = node_text(node)
|
| 78 |
+
if len(body) < MIN_CHUNK_CHARS:
|
| 79 |
+
covered.update(range(start, end + 1))
|
| 80 |
+
return
|
| 81 |
+
chunks.append({
|
| 82 |
+
"id": f"{rel_path}::{symbol}::L{start}",
|
| 83 |
+
"text": header + body,
|
| 84 |
+
"source_type": "code",
|
| 85 |
+
"path": rel_path,
|
| 86 |
+
"symbol": symbol,
|
| 87 |
+
"url": github_url(repo, branch, rel_path, start, end),
|
| 88 |
+
"start_line": start,
|
| 89 |
+
"end_line": end,
|
| 90 |
+
})
|
| 91 |
+
covered.update(range(start, end + 1))
|
| 92 |
+
|
| 93 |
+
def unwrap(node):
|
| 94 |
+
"""decorated_definition wraps the real def; we want the decorator
|
| 95 |
+
text included but the inner node's name."""
|
| 96 |
+
if node.type == "decorated_definition":
|
| 97 |
+
for child in node.children:
|
| 98 |
+
if child.type in ("function_definition", "class_definition"):
|
| 99 |
+
return child, node
|
| 100 |
+
return node, node
|
| 101 |
+
|
| 102 |
+
def symbol_name(def_node) -> str:
|
| 103 |
+
name = def_node.child_by_field_name("name")
|
| 104 |
+
return name.text.decode() if name else "?"
|
| 105 |
+
|
| 106 |
+
for node in tree.root_node.children:
|
| 107 |
+
inner, outer = unwrap(node)
|
| 108 |
+
|
| 109 |
+
if inner.type == "function_definition":
|
| 110 |
+
emit(outer, symbol_name(inner))
|
| 111 |
+
|
| 112 |
+
elif inner.type == "class_definition":
|
| 113 |
+
cls_name = symbol_name(inner)
|
| 114 |
+
n_lines = inner.end_point[0] - inner.start_point[0] + 1
|
| 115 |
+
if n_lines <= MAX_CLASS_LINES:
|
| 116 |
+
emit(outer, cls_name)
|
| 117 |
+
else:
|
| 118 |
+
# Big class: one chunk per method
|
| 119 |
+
body = inner.child_by_field_name("body")
|
| 120 |
+
for child in (body.children if body else []):
|
| 121 |
+
m_inner, m_outer = unwrap(child)
|
| 122 |
+
if m_inner.type == "function_definition":
|
| 123 |
+
emit(m_outer, f"{cls_name}.{symbol_name(m_inner)}",
|
| 124 |
+
context=f"Class: {cls_name}")
|
| 125 |
+
|
| 126 |
+
# Module-level leftovers: imports, constants, top-level statements
|
| 127 |
+
leftover = [i for i in range(1, len(text_lines) + 1)
|
| 128 |
+
if i not in covered and text_lines[i - 1].strip()]
|
| 129 |
+
if leftover:
|
| 130 |
+
body = "\n".join(text_lines[i - 1] for i in leftover)
|
| 131 |
+
if len(body) >= MIN_CHUNK_CHARS:
|
| 132 |
+
start, end = leftover[0], leftover[-1]
|
| 133 |
+
chunks.append({
|
| 134 |
+
"id": f"{rel_path}::<module>",
|
| 135 |
+
"text": f"# File: {rel_path} | Symbol: <module>\n{body}",
|
| 136 |
+
"source_type": "code",
|
| 137 |
+
"path": rel_path,
|
| 138 |
+
"symbol": "<module>",
|
| 139 |
+
"url": github_url(repo, branch, rel_path, start, end),
|
| 140 |
+
"start_line": start,
|
| 141 |
+
"end_line": end,
|
| 142 |
+
})
|
| 143 |
+
return chunks
|
| 144 |
+
|
| 145 |
+
|
| 146 |
+
# ---------------------------------------------------------------- docs
|
| 147 |
+
|
| 148 |
+
def chunk_markdown_file(text: str, rel_path: str, repo: str,
|
| 149 |
+
branch: str) -> list[dict]:
|
| 150 |
+
"""Split on '## ' headings; each section keeps the page title as context."""
|
| 151 |
+
lines = text.splitlines()
|
| 152 |
+
page_title = next((l.lstrip("# ").strip() for l in lines
|
| 153 |
+
if l.startswith("# ")), rel_path)
|
| 154 |
+
|
| 155 |
+
sections: list[tuple[str, int]] = [] # (heading, start_line 1-based)
|
| 156 |
+
for i, line in enumerate(lines):
|
| 157 |
+
if line.startswith("## "):
|
| 158 |
+
sections.append((line.lstrip("# ").strip(), i + 1))
|
| 159 |
+
if not sections or sections[0][1] > 1:
|
| 160 |
+
sections.insert(0, (page_title, 1))
|
| 161 |
+
|
| 162 |
+
chunks = []
|
| 163 |
+
for idx, (heading, start) in enumerate(sections):
|
| 164 |
+
end = sections[idx + 1][1] - 1 if idx + 1 < len(sections) else len(lines)
|
| 165 |
+
body = "\n".join(lines[start - 1:end]).strip()
|
| 166 |
+
if len(body) < MIN_CHUNK_CHARS:
|
| 167 |
+
continue
|
| 168 |
+
header = f"# Doc: {rel_path} | Page: {page_title} | Section: {heading}\n"
|
| 169 |
+
chunks.append({
|
| 170 |
+
"id": f"{rel_path}::{heading}::L{start}",
|
| 171 |
+
"text": header + body,
|
| 172 |
+
"source_type": "doc",
|
| 173 |
+
"path": rel_path,
|
| 174 |
+
"symbol": heading,
|
| 175 |
+
"url": github_url(repo, branch, rel_path, start, end),
|
| 176 |
+
"start_line": start,
|
| 177 |
+
"end_line": end,
|
| 178 |
+
})
|
| 179 |
+
return chunks
|
| 180 |
+
|
| 181 |
+
|
| 182 |
+
# ---------------------------------------------------------------- issues
|
| 183 |
+
|
| 184 |
+
def chunk_issue(record: dict) -> dict | None:
|
| 185 |
+
"""One chunk per issue: title + body + top comments."""
|
| 186 |
+
parts = [f"# Issue #{record['number']}: {record['title']}"]
|
| 187 |
+
if record.get("labels"):
|
| 188 |
+
parts.append(f"Labels: {', '.join(record['labels'])}")
|
| 189 |
+
if record.get("body"):
|
| 190 |
+
parts.append(record["body"][:MAX_ISSUE_BODY])
|
| 191 |
+
for c in record.get("comments", []):
|
| 192 |
+
parts.append(f"---\nComment: {c[:MAX_COMMENT]}")
|
| 193 |
+
text = "\n".join(parts)
|
| 194 |
+
if len(text) < MIN_CHUNK_CHARS:
|
| 195 |
+
return None
|
| 196 |
+
return {
|
| 197 |
+
"id": f"issue::{record['number']}",
|
| 198 |
+
"text": text,
|
| 199 |
+
"source_type": "issue",
|
| 200 |
+
"path": f"issues/{record['number']}.json",
|
| 201 |
+
"symbol": record["title"],
|
| 202 |
+
"url": record["url"],
|
| 203 |
+
"start_line": None,
|
| 204 |
+
"end_line": None,
|
| 205 |
+
}
|
| 206 |
+
|
| 207 |
+
|
| 208 |
+
# ---------------------------------------------------------------- main
|
| 209 |
+
|
| 210 |
+
def main() -> None:
|
| 211 |
+
manifest = json.loads((DATA_DIR / "manifest.json").read_text())
|
| 212 |
+
repo = manifest["repo"]
|
| 213 |
+
branch = default_branch()
|
| 214 |
+
|
| 215 |
+
all_chunks: list[dict] = []
|
| 216 |
+
|
| 217 |
+
for rec in manifest["files"]:
|
| 218 |
+
path = REPO_DIR / rec["path"]
|
| 219 |
+
if rec["source_type"] == "code":
|
| 220 |
+
all_chunks += chunk_code_file(path.read_bytes(), rec["path"],
|
| 221 |
+
repo, branch)
|
| 222 |
+
elif rec["source_type"] == "doc":
|
| 223 |
+
all_chunks += chunk_markdown_file(
|
| 224 |
+
path.read_text(errors="replace"), rec["path"], repo, branch)
|
| 225 |
+
|
| 226 |
+
for issue_file in sorted((DATA_DIR / "issues").glob("*.json")):
|
| 227 |
+
chunk = chunk_issue(json.loads(issue_file.read_text()))
|
| 228 |
+
if chunk:
|
| 229 |
+
all_chunks.append(chunk)
|
| 230 |
+
|
| 231 |
+
out = DATA_DIR / "chunks.jsonl"
|
| 232 |
+
with out.open("w", encoding="utf-8") as f:
|
| 233 |
+
for c in all_chunks:
|
| 234 |
+
f.write(json.dumps(c, ensure_ascii=False) + "\n")
|
| 235 |
+
|
| 236 |
+
counts: dict[str, int] = {}
|
| 237 |
+
sizes = []
|
| 238 |
+
for c in all_chunks:
|
| 239 |
+
counts[c["source_type"]] = counts.get(c["source_type"], 0) + 1
|
| 240 |
+
sizes.append(len(c["text"]))
|
| 241 |
+
print(f"[done] {len(all_chunks)} chunks -> {out}")
|
| 242 |
+
print(f"[done] by type: {counts}")
|
| 243 |
+
print(f"[done] chars/chunk: min={min(sizes)} "
|
| 244 |
+
f"avg={sum(sizes)//len(sizes)} max={max(sizes)}")
|
| 245 |
+
|
| 246 |
+
|
| 247 |
+
if __name__ == "__main__":
|
| 248 |
+
main()
|
tests/__pycache__/test_chunk.cpython-311-pytest-9.1.1.pyc
ADDED
|
Binary file (14 kB). View file
|
|
|
tests/test_chunk.py
ADDED
|
@@ -0,0 +1,114 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Tests for the chunkers. Run with: pytest tests/ -v"""
|
| 2 |
+
|
| 3 |
+
import sys
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
|
| 6 |
+
# Make src/ importable when running pytest from the project root
|
| 7 |
+
sys.path.insert(0, str(Path(__file__).parent.parent / "src"))
|
| 8 |
+
|
| 9 |
+
from chunk import chunk_code_file, chunk_issue, chunk_markdown_file
|
| 10 |
+
|
| 11 |
+
SAMPLE_CODE = b'''\
|
| 12 |
+
import os
|
| 13 |
+
|
| 14 |
+
CONSTANT_VALUE = "something important that should end up in the module chunk"
|
| 15 |
+
|
| 16 |
+
def greet(name: str) -> str:
|
| 17 |
+
"""Say hello to someone by name, politely and at length."""
|
| 18 |
+
message = f"Hello, {name}! Welcome to the test suite."
|
| 19 |
+
return message
|
| 20 |
+
|
| 21 |
+
@property
|
| 22 |
+
def decorated_function(self):
|
| 23 |
+
"""A decorated function to test decorated_definition unwrapping."""
|
| 24 |
+
return self._value
|
| 25 |
+
|
| 26 |
+
class Greeter:
|
| 27 |
+
"""A small class that fits in one chunk."""
|
| 28 |
+
|
| 29 |
+
def __init__(self, name: str) -> None:
|
| 30 |
+
self.name = name
|
| 31 |
+
|
| 32 |
+
def greet(self) -> str:
|
| 33 |
+
return f"Hello, {self.name}!"
|
| 34 |
+
'''
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def test_function_becomes_chunk():
|
| 38 |
+
chunks = chunk_code_file(SAMPLE_CODE, "pkg/mod.py", "owner/repo", "main")
|
| 39 |
+
symbols = [c["symbol"] for c in chunks]
|
| 40 |
+
assert "greet" in symbols
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
def test_chunk_has_context_header():
|
| 44 |
+
chunks = chunk_code_file(SAMPLE_CODE, "pkg/mod.py", "owner/repo", "main")
|
| 45 |
+
greet = next(c for c in chunks if c["symbol"] == "greet")
|
| 46 |
+
assert greet["text"].startswith("# File: pkg/mod.py")
|
| 47 |
+
assert "def greet" in greet["text"]
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def test_line_numbers_are_correct():
|
| 51 |
+
chunks = chunk_code_file(SAMPLE_CODE, "pkg/mod.py", "owner/repo", "main")
|
| 52 |
+
greet = next(c for c in chunks if c["symbol"] == "greet")
|
| 53 |
+
lines = SAMPLE_CODE.decode().splitlines()
|
| 54 |
+
assert lines[greet["start_line"] - 1].startswith("def greet")
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
def test_class_kept_whole_when_small():
|
| 58 |
+
chunks = chunk_code_file(SAMPLE_CODE, "pkg/mod.py", "owner/repo", "main")
|
| 59 |
+
assert any(c["symbol"] == "Greeter" for c in chunks)
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
def test_module_chunk_collects_constants():
|
| 63 |
+
chunks = chunk_code_file(SAMPLE_CODE, "pkg/mod.py", "owner/repo", "main")
|
| 64 |
+
module = next(c for c in chunks if c["symbol"] == "<module>")
|
| 65 |
+
assert "CONSTANT_VALUE" in module["text"]
|
| 66 |
+
|
| 67 |
+
|
| 68 |
+
def test_no_empty_chunks():
|
| 69 |
+
chunks = chunk_code_file(SAMPLE_CODE, "pkg/mod.py", "owner/repo", "main")
|
| 70 |
+
assert all(c["text"].strip() for c in chunks)
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
SAMPLE_MD = """\
|
| 74 |
+
# Getting Started
|
| 75 |
+
|
| 76 |
+
Some introduction text that is long enough to keep as a chunk of its own.
|
| 77 |
+
|
| 78 |
+
## Installation
|
| 79 |
+
|
| 80 |
+
Run pip install and you are ready to go. This section explains the details
|
| 81 |
+
of installing the package in a virtual environment.
|
| 82 |
+
|
| 83 |
+
## Usage
|
| 84 |
+
|
| 85 |
+
Import the package and call the main function to get going with the API.
|
| 86 |
+
"""
|
| 87 |
+
|
| 88 |
+
|
| 89 |
+
def test_markdown_splits_on_h2():
|
| 90 |
+
chunks = chunk_markdown_file(SAMPLE_MD, "docs/en/start.md", "o/r", "main")
|
| 91 |
+
sections = [c["symbol"] for c in chunks]
|
| 92 |
+
assert "Installation" in sections
|
| 93 |
+
assert "Usage" in sections
|
| 94 |
+
|
| 95 |
+
|
| 96 |
+
def test_markdown_keeps_page_title_context():
|
| 97 |
+
chunks = chunk_markdown_file(SAMPLE_MD, "docs/en/start.md", "o/r", "main")
|
| 98 |
+
install = next(c for c in chunks if c["symbol"] == "Installation")
|
| 99 |
+
assert "Getting Started" in install["text"]
|
| 100 |
+
|
| 101 |
+
|
| 102 |
+
def test_issue_chunk():
|
| 103 |
+
record = {
|
| 104 |
+
"number": 42,
|
| 105 |
+
"title": "App crashes on startup",
|
| 106 |
+
"body": "When I run the app it crashes immediately with a traceback.",
|
| 107 |
+
"labels": ["bug"],
|
| 108 |
+
"url": "https://github.com/o/r/issues/42",
|
| 109 |
+
"comments": ["Fixed by upgrading the dependency to version 2.0."],
|
| 110 |
+
}
|
| 111 |
+
chunk = chunk_issue(record)
|
| 112 |
+
assert chunk["id"] == "issue::42"
|
| 113 |
+
assert "crashes" in chunk["text"]
|
| 114 |
+
assert "Fixed by upgrading" in chunk["text"]
|