Qurrah commited on
Commit
bf80b95
·
1 Parent(s): 6657fae

ADD: implement text chunking functionality for policy content processing

Browse files
Files changed (1) hide show
  1. rag/chunker.py +27 -0
rag/chunker.py ADDED
@@ -0,0 +1,27 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from langchain_text_splitters import RecursiveCharacterTextSplitter
2
+
3
+
4
+ splitter = RecursiveCharacterTextSplitter(
5
+ chunk_size=500,
6
+ chunk_overlap=50,
7
+ separators=["\n\n", "\n", ".", "!", "?", ",", " "],
8
+ length_function=len,
9
+ )
10
+
11
+
12
+ def prepare_policy_chunks(policy: dict) -> list[dict]:
13
+ chunks = splitter.split_text(policy["content"])
14
+ return [
15
+ {
16
+ "workspace_id": policy["workspace_id"],
17
+ "source_type": "policy",
18
+ "source_id": policy["id"],
19
+ "chunk_index": i,
20
+ "content": chunk,
21
+ "metadata": {
22
+ "policy_type": policy["type"],
23
+ "policy_title": policy["title"],
24
+ },
25
+ }
26
+ for i, chunk in enumerate(chunks)
27
+ ]