Islam Mamedov commited on
Commit
98aa8c9
·
1 Parent(s): 3df8a15

Day 2: AST-aware chunkers for code, docs, issues

Browse files
data/chunks.jsonl ADDED
The diff for this file is too large to render. See raw diff
 
src/__pycache__/chunk.cpython-311.pyc ADDED
Binary file (14.2 kB). View file
 
src/chunk.py ADDED
@@ -0,0 +1,248 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Chunk ingested content into retrieval units.
2
+
3
+ Reads data/manifest.json + files produced by ingest.py.
4
+ Writes data/chunks.jsonl — one JSON chunk per line.
5
+
6
+ Chunk schema (same for all source types):
7
+ id stable unique id
8
+ text what gets embedded (includes a context header)
9
+ source_type code | doc | issue
10
+ path file path in the repo (or issues/N.json)
11
+ symbol function/class name, doc heading, or issue title
12
+ url link to the exact GitHub location
13
+ start_line / end_line (code and docs only)
14
+
15
+ Usage:
16
+ python src/chunk.py
17
+ """
18
+
19
+ import json
20
+ import subprocess
21
+ from pathlib import Path
22
+
23
+ import tree_sitter_python as tspython
24
+ from tree_sitter import Language, Parser
25
+
26
+ DATA_DIR = Path("data")
27
+ REPO_DIR = DATA_DIR / "repo"
28
+ MAX_CLASS_LINES = 150 # classes bigger than this get split into methods
29
+ MIN_CHUNK_CHARS = 50 # drop trivial fragments
30
+ MAX_ISSUE_BODY = 3000
31
+ MAX_COMMENT = 1000
32
+
33
+ PY_LANGUAGE = Language(tspython.language())
34
+ PARSER = Parser(PY_LANGUAGE)
35
+
36
+
37
+ # ---------------------------------------------------------------- helpers
38
+
39
+ def default_branch() -> str:
40
+ out = subprocess.run(["git", "-C", str(REPO_DIR), "rev-parse",
41
+ "--abbrev-ref", "HEAD"],
42
+ capture_output=True, text=True)
43
+ return out.stdout.strip() or "master"
44
+
45
+
46
+ def github_url(repo: str, branch: str, path: str,
47
+ start: int | None = None, end: int | None = None) -> str:
48
+ url = f"https://github.com/{repo}/blob/{branch}/{path}"
49
+ if start is not None:
50
+ url += f"#L{start}-L{end}"
51
+ return url
52
+
53
+
54
+ # ---------------------------------------------------------------- code
55
+
56
+ def chunk_code_file(source: bytes, rel_path: str, repo: str,
57
+ branch: str) -> list[dict]:
58
+ """AST-aware chunking: one chunk per top-level function/class.
59
+
60
+ Large classes are split into per-method chunks. Module-level leftovers
61
+ (imports, constants) become one '<module>' chunk.
62
+ """
63
+ tree = PARSER.parse(source)
64
+ text_lines = source.decode(errors="replace").splitlines()
65
+ chunks: list[dict] = []
66
+ covered: set[int] = set() # 1-based line numbers already in a chunk
67
+
68
+ def node_text(node) -> str:
69
+ return source[node.start_byte:node.end_byte].decode(errors="replace")
70
+
71
+ def emit(node, symbol: str, context: str = "") -> None:
72
+ start, end = node.start_point[0] + 1, node.end_point[0] + 1
73
+ header = f"# File: {rel_path}"
74
+ if context:
75
+ header += f" | {context}"
76
+ header += f" | Symbol: {symbol}\n"
77
+ body = node_text(node)
78
+ if len(body) < MIN_CHUNK_CHARS:
79
+ covered.update(range(start, end + 1))
80
+ return
81
+ chunks.append({
82
+ "id": f"{rel_path}::{symbol}::L{start}",
83
+ "text": header + body,
84
+ "source_type": "code",
85
+ "path": rel_path,
86
+ "symbol": symbol,
87
+ "url": github_url(repo, branch, rel_path, start, end),
88
+ "start_line": start,
89
+ "end_line": end,
90
+ })
91
+ covered.update(range(start, end + 1))
92
+
93
+ def unwrap(node):
94
+ """decorated_definition wraps the real def; we want the decorator
95
+ text included but the inner node's name."""
96
+ if node.type == "decorated_definition":
97
+ for child in node.children:
98
+ if child.type in ("function_definition", "class_definition"):
99
+ return child, node
100
+ return node, node
101
+
102
+ def symbol_name(def_node) -> str:
103
+ name = def_node.child_by_field_name("name")
104
+ return name.text.decode() if name else "?"
105
+
106
+ for node in tree.root_node.children:
107
+ inner, outer = unwrap(node)
108
+
109
+ if inner.type == "function_definition":
110
+ emit(outer, symbol_name(inner))
111
+
112
+ elif inner.type == "class_definition":
113
+ cls_name = symbol_name(inner)
114
+ n_lines = inner.end_point[0] - inner.start_point[0] + 1
115
+ if n_lines <= MAX_CLASS_LINES:
116
+ emit(outer, cls_name)
117
+ else:
118
+ # Big class: one chunk per method
119
+ body = inner.child_by_field_name("body")
120
+ for child in (body.children if body else []):
121
+ m_inner, m_outer = unwrap(child)
122
+ if m_inner.type == "function_definition":
123
+ emit(m_outer, f"{cls_name}.{symbol_name(m_inner)}",
124
+ context=f"Class: {cls_name}")
125
+
126
+ # Module-level leftovers: imports, constants, top-level statements
127
+ leftover = [i for i in range(1, len(text_lines) + 1)
128
+ if i not in covered and text_lines[i - 1].strip()]
129
+ if leftover:
130
+ body = "\n".join(text_lines[i - 1] for i in leftover)
131
+ if len(body) >= MIN_CHUNK_CHARS:
132
+ start, end = leftover[0], leftover[-1]
133
+ chunks.append({
134
+ "id": f"{rel_path}::<module>",
135
+ "text": f"# File: {rel_path} | Symbol: <module>\n{body}",
136
+ "source_type": "code",
137
+ "path": rel_path,
138
+ "symbol": "<module>",
139
+ "url": github_url(repo, branch, rel_path, start, end),
140
+ "start_line": start,
141
+ "end_line": end,
142
+ })
143
+ return chunks
144
+
145
+
146
+ # ---------------------------------------------------------------- docs
147
+
148
+ def chunk_markdown_file(text: str, rel_path: str, repo: str,
149
+ branch: str) -> list[dict]:
150
+ """Split on '## ' headings; each section keeps the page title as context."""
151
+ lines = text.splitlines()
152
+ page_title = next((l.lstrip("# ").strip() for l in lines
153
+ if l.startswith("# ")), rel_path)
154
+
155
+ sections: list[tuple[str, int]] = [] # (heading, start_line 1-based)
156
+ for i, line in enumerate(lines):
157
+ if line.startswith("## "):
158
+ sections.append((line.lstrip("# ").strip(), i + 1))
159
+ if not sections or sections[0][1] > 1:
160
+ sections.insert(0, (page_title, 1))
161
+
162
+ chunks = []
163
+ for idx, (heading, start) in enumerate(sections):
164
+ end = sections[idx + 1][1] - 1 if idx + 1 < len(sections) else len(lines)
165
+ body = "\n".join(lines[start - 1:end]).strip()
166
+ if len(body) < MIN_CHUNK_CHARS:
167
+ continue
168
+ header = f"# Doc: {rel_path} | Page: {page_title} | Section: {heading}\n"
169
+ chunks.append({
170
+ "id": f"{rel_path}::{heading}::L{start}",
171
+ "text": header + body,
172
+ "source_type": "doc",
173
+ "path": rel_path,
174
+ "symbol": heading,
175
+ "url": github_url(repo, branch, rel_path, start, end),
176
+ "start_line": start,
177
+ "end_line": end,
178
+ })
179
+ return chunks
180
+
181
+
182
+ # ---------------------------------------------------------------- issues
183
+
184
+ def chunk_issue(record: dict) -> dict | None:
185
+ """One chunk per issue: title + body + top comments."""
186
+ parts = [f"# Issue #{record['number']}: {record['title']}"]
187
+ if record.get("labels"):
188
+ parts.append(f"Labels: {', '.join(record['labels'])}")
189
+ if record.get("body"):
190
+ parts.append(record["body"][:MAX_ISSUE_BODY])
191
+ for c in record.get("comments", []):
192
+ parts.append(f"---\nComment: {c[:MAX_COMMENT]}")
193
+ text = "\n".join(parts)
194
+ if len(text) < MIN_CHUNK_CHARS:
195
+ return None
196
+ return {
197
+ "id": f"issue::{record['number']}",
198
+ "text": text,
199
+ "source_type": "issue",
200
+ "path": f"issues/{record['number']}.json",
201
+ "symbol": record["title"],
202
+ "url": record["url"],
203
+ "start_line": None,
204
+ "end_line": None,
205
+ }
206
+
207
+
208
+ # ---------------------------------------------------------------- main
209
+
210
+ def main() -> None:
211
+ manifest = json.loads((DATA_DIR / "manifest.json").read_text())
212
+ repo = manifest["repo"]
213
+ branch = default_branch()
214
+
215
+ all_chunks: list[dict] = []
216
+
217
+ for rec in manifest["files"]:
218
+ path = REPO_DIR / rec["path"]
219
+ if rec["source_type"] == "code":
220
+ all_chunks += chunk_code_file(path.read_bytes(), rec["path"],
221
+ repo, branch)
222
+ elif rec["source_type"] == "doc":
223
+ all_chunks += chunk_markdown_file(
224
+ path.read_text(errors="replace"), rec["path"], repo, branch)
225
+
226
+ for issue_file in sorted((DATA_DIR / "issues").glob("*.json")):
227
+ chunk = chunk_issue(json.loads(issue_file.read_text()))
228
+ if chunk:
229
+ all_chunks.append(chunk)
230
+
231
+ out = DATA_DIR / "chunks.jsonl"
232
+ with out.open("w", encoding="utf-8") as f:
233
+ for c in all_chunks:
234
+ f.write(json.dumps(c, ensure_ascii=False) + "\n")
235
+
236
+ counts: dict[str, int] = {}
237
+ sizes = []
238
+ for c in all_chunks:
239
+ counts[c["source_type"]] = counts.get(c["source_type"], 0) + 1
240
+ sizes.append(len(c["text"]))
241
+ print(f"[done] {len(all_chunks)} chunks -> {out}")
242
+ print(f"[done] by type: {counts}")
243
+ print(f"[done] chars/chunk: min={min(sizes)} "
244
+ f"avg={sum(sizes)//len(sizes)} max={max(sizes)}")
245
+
246
+
247
+ if __name__ == "__main__":
248
+ main()
tests/__pycache__/test_chunk.cpython-311-pytest-9.1.1.pyc ADDED
Binary file (14 kB). View file
 
tests/test_chunk.py ADDED
@@ -0,0 +1,114 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Tests for the chunkers. Run with: pytest tests/ -v"""
2
+
3
+ import sys
4
+ from pathlib import Path
5
+
6
+ # Make src/ importable when running pytest from the project root
7
+ sys.path.insert(0, str(Path(__file__).parent.parent / "src"))
8
+
9
+ from chunk import chunk_code_file, chunk_issue, chunk_markdown_file
10
+
11
+ SAMPLE_CODE = b'''\
12
+ import os
13
+
14
+ CONSTANT_VALUE = "something important that should end up in the module chunk"
15
+
16
+ def greet(name: str) -> str:
17
+ """Say hello to someone by name, politely and at length."""
18
+ message = f"Hello, {name}! Welcome to the test suite."
19
+ return message
20
+
21
+ @property
22
+ def decorated_function(self):
23
+ """A decorated function to test decorated_definition unwrapping."""
24
+ return self._value
25
+
26
+ class Greeter:
27
+ """A small class that fits in one chunk."""
28
+
29
+ def __init__(self, name: str) -> None:
30
+ self.name = name
31
+
32
+ def greet(self) -> str:
33
+ return f"Hello, {self.name}!"
34
+ '''
35
+
36
+
37
+ def test_function_becomes_chunk():
38
+ chunks = chunk_code_file(SAMPLE_CODE, "pkg/mod.py", "owner/repo", "main")
39
+ symbols = [c["symbol"] for c in chunks]
40
+ assert "greet" in symbols
41
+
42
+
43
+ def test_chunk_has_context_header():
44
+ chunks = chunk_code_file(SAMPLE_CODE, "pkg/mod.py", "owner/repo", "main")
45
+ greet = next(c for c in chunks if c["symbol"] == "greet")
46
+ assert greet["text"].startswith("# File: pkg/mod.py")
47
+ assert "def greet" in greet["text"]
48
+
49
+
50
+ def test_line_numbers_are_correct():
51
+ chunks = chunk_code_file(SAMPLE_CODE, "pkg/mod.py", "owner/repo", "main")
52
+ greet = next(c for c in chunks if c["symbol"] == "greet")
53
+ lines = SAMPLE_CODE.decode().splitlines()
54
+ assert lines[greet["start_line"] - 1].startswith("def greet")
55
+
56
+
57
+ def test_class_kept_whole_when_small():
58
+ chunks = chunk_code_file(SAMPLE_CODE, "pkg/mod.py", "owner/repo", "main")
59
+ assert any(c["symbol"] == "Greeter" for c in chunks)
60
+
61
+
62
+ def test_module_chunk_collects_constants():
63
+ chunks = chunk_code_file(SAMPLE_CODE, "pkg/mod.py", "owner/repo", "main")
64
+ module = next(c for c in chunks if c["symbol"] == "<module>")
65
+ assert "CONSTANT_VALUE" in module["text"]
66
+
67
+
68
+ def test_no_empty_chunks():
69
+ chunks = chunk_code_file(SAMPLE_CODE, "pkg/mod.py", "owner/repo", "main")
70
+ assert all(c["text"].strip() for c in chunks)
71
+
72
+
73
+ SAMPLE_MD = """\
74
+ # Getting Started
75
+
76
+ Some introduction text that is long enough to keep as a chunk of its own.
77
+
78
+ ## Installation
79
+
80
+ Run pip install and you are ready to go. This section explains the details
81
+ of installing the package in a virtual environment.
82
+
83
+ ## Usage
84
+
85
+ Import the package and call the main function to get going with the API.
86
+ """
87
+
88
+
89
+ def test_markdown_splits_on_h2():
90
+ chunks = chunk_markdown_file(SAMPLE_MD, "docs/en/start.md", "o/r", "main")
91
+ sections = [c["symbol"] for c in chunks]
92
+ assert "Installation" in sections
93
+ assert "Usage" in sections
94
+
95
+
96
+ def test_markdown_keeps_page_title_context():
97
+ chunks = chunk_markdown_file(SAMPLE_MD, "docs/en/start.md", "o/r", "main")
98
+ install = next(c for c in chunks if c["symbol"] == "Installation")
99
+ assert "Getting Started" in install["text"]
100
+
101
+
102
+ def test_issue_chunk():
103
+ record = {
104
+ "number": 42,
105
+ "title": "App crashes on startup",
106
+ "body": "When I run the app it crashes immediately with a traceback.",
107
+ "labels": ["bug"],
108
+ "url": "https://github.com/o/r/issues/42",
109
+ "comments": ["Fixed by upgrading the dependency to version 2.0."],
110
+ }
111
+ chunk = chunk_issue(record)
112
+ assert chunk["id"] == "issue::42"
113
+ assert "crashes" in chunk["text"]
114
+ assert "Fixed by upgrading" in chunk["text"]