Spaces:

AI-Driven-Data-Driven
/

Arabic-Rag-Chatbot

Sleeping

Arabic-Rag-Chatbot / document_processor.py

Update document_processor.py

99080be verified about 1 year ago

2.86 kB

# document_processor.py
import os
import glob
from tqdm import tqdm
import pandas as pd
from utils import clean_text, setup_logger

logger = setup_logger('document_processor')

def split_into_chunks(text, chunk_size=300, overlap=100):
    """
    Split text into overlapping chunks
    
    Args:
        text: The text to split
        chunk_size: Number of characters per chunk
        overlap: Number of characters to overlap between chunks
    """
    chunks = []
    start = 0
    text_length = len(text)
    
    while start < text_length:
        end = start + chunk_size
        chunk = text[start:end]
        
        # Try to break at sentence boundary for better context
        if end < text_length:
            # Look for sentence endings
            last_period = chunk.rfind('.')
            last_question = chunk.rfind('؟')  # Arabic question mark
            last_exclamation = chunk.rfind('!')
            last_newline = chunk.rfind('\n')
            
            # Find the best break point
            break_point = max(last_period, last_question, last_exclamation, last_newline)
            
            # Only break if we're past halfway through the chunk
            if break_point > chunk_size * 0.5:
                chunk = chunk[:break_point + 1]
                end = start + break_point + 1
        
        chunk = chunk.strip()
        if chunk:  # Only add non-empty chunks
            chunks.append(chunk)
        
        start = end - overlap  # Move start with overlap
    
    return chunks



def load_single_document(file_path, chunk_size=300, overlap=100):
    """
    Load a single document and split it into chunks
    
    Args:
        file_path: Path to the .txt file
        chunk_size: Size of each chunk in characters
        overlap: Overlap between chunks in characters
    """
    try:
        with open(file_path, 'r', encoding='utf-8') as file:
            content = clean_text(file.read())
            
            if not content:
                logger.warning(f"Empty content in {file_path}")
                return pd.DataFrame()
            
            # Split into chunks
            chunks = split_into_chunks(content, chunk_size, overlap)
            
            # Create dataframe with chunks
            documents = []
            for i, chunk in enumerate(chunks):
                documents.append({
                    'path': file_path,
                    'chunk_id': i,
                    'total_chunks': len(chunks),
                    'content': chunk,
                    'content_length': len(chunk)
                })
            
            logger.info(f"Loaded {os.path.basename(file_path)}: {len(chunks)} chunks")
            return pd.DataFrame(documents)
            
    except Exception as e:
        logger.error(f"Error reading {file_path}: {e}")
        return pd.DataFrame()

	# document_processor.py
	import os
	import glob
	from tqdm import tqdm
	import pandas as pd
	from utils import clean_text, setup_logger

	logger = setup_logger('document_processor')

	def split_into_chunks(text, chunk_size=300, overlap=100):
	"""
	Split text into overlapping chunks

	Args:
	text: The text to split
	chunk_size: Number of characters per chunk
	overlap: Number of characters to overlap between chunks
	"""
	chunks = []
	start = 0
	text_length = len(text)

	while start < text_length:
	end = start + chunk_size
	chunk = text[start:end]

	# Try to break at sentence boundary for better context
	if end < text_length:
	# Look for sentence endings
	last_period = chunk.rfind('.')
	last_question = chunk.rfind('؟') # Arabic question mark
	last_exclamation = chunk.rfind('!')
	last_newline = chunk.rfind('\n')

	# Find the best break point
	break_point = max(last_period, last_question, last_exclamation, last_newline)

	# Only break if we're past halfway through the chunk
	if break_point > chunk_size * 0.5:
	chunk = chunk[:break_point + 1]
	end = start + break_point + 1

	chunk = chunk.strip()
	if chunk: # Only add non-empty chunks
	chunks.append(chunk)

	start = end - overlap # Move start with overlap

	return chunks



	def load_single_document(file_path, chunk_size=300, overlap=100):
	"""
	Load a single document and split it into chunks

	Args:
	file_path: Path to the .txt file
	chunk_size: Size of each chunk in characters
	overlap: Overlap between chunks in characters
	"""
	try:
	with open(file_path, 'r', encoding='utf-8') as file:
	content = clean_text(file.read())

	if not content:
	logger.warning(f"Empty content in {file_path}")
	return pd.DataFrame()

	# Split into chunks
	chunks = split_into_chunks(content, chunk_size, overlap)

	# Create dataframe with chunks
	documents = []
	for i, chunk in enumerate(chunks):
	documents.append({
	'path': file_path,
	'chunk_id': i,
	'total_chunks': len(chunks),
	'content': chunk,
	'content_length': len(chunk)
	})

	logger.info(f"Loaded {os.path.basename(file_path)}: {len(chunks)} chunks")
	return pd.DataFrame(documents)

	except Exception as e:
	logger.error(f"Error reading {file_path}: {e}")
	return pd.DataFrame()