File size: 1,157 Bytes
ceabc0d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
from langchain_huggingface import HuggingFaceEmbeddings
from langchain.text_splitter import RecursiveCharacterTextSplitter, CharacterTextSplitter, TokenTextSplitter

def setup_chunking_and_embedding(documents, chunking_strategy, chunk_size, chunk_overlap, embedding_model):
    """Setup text chunking and embedding."""
    embedding_models = {
        'HuggingFace': HuggingFaceEmbeddings()
    }
    
    embeddings = embedding_models.get(embedding_model)
    if not embeddings:
        raise ValueError("Unsupported embedding model.")

    chunking_strategies = {
        'Recursive': RecursiveCharacterTextSplitter,
        'Character': CharacterTextSplitter,
        'Token': TokenTextSplitter
    }

    text_splitter = chunking_strategies.get(chunking_strategy)
    if not text_splitter:
        raise ValueError("Unsupported chunking strategy.")
    
    splitter_instance = text_splitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap)
    doc_chunks = splitter_instance.split_documents(documents)

    if not doc_chunks:
        raise ValueError("No document chunks created.")
    
    return doc_chunks, embeddings