Spaces:

nitinvig
/

HindiBPE

Sleeping

HindiBPE / hindi_preprocessor.py

Upload 13 files

e055374 verified 11 months ago

9.57 kB

# ============================================================================
# HINDI TEXT PREPROCESSOR (OPTIMIZED FOR PYTHON 3.11)
# ============================================================================
# This module contains regex-based preprocessing functions for Hindi text
# Handles Devanagari script patterns, punctuation, and normalization
# Optimized for large datasets with compiled regex patterns

import re
from typing import Iterator, Optional


# Devanagari Unicode ranges
# Devanagari: U+0900 to U+097F
# Devanagari Extended: U+A8E0 to U+A8FF
DEVANAGARI_PATTERN = r'[\u0900-\u097F\uA8E0-\uA8FF]+'

# Common Hindi punctuation marks
HINDI_PUNCTUATION = r'[।॥,;:!?\-—–()\[\]{}"\'\'""]'

# Numbers (both Devanagari and Arabic)
DEVANAGARI_NUMBERS = r'[\u0966-\u096F]'  # ०-९
ARABIC_NUMBERS = r'[0-9]'

# Pre-compile regex patterns for better performance (Python 3.11 optimization)
# Compiling once and reusing is much faster than compiling on each call
_WHITESPACE_PATTERN = re.compile(r'\s+')
_SPACE_BEFORE_PUNCT = re.compile(r'\s+([,;:!?।॥])')
_PUNCT_AFTER_SPACE = re.compile(r'([,;:!?।॥])([^\s])')
_OPEN_QUOTE_SPACE = re.compile(r'([""\'\([{])\s+')
_CLOSE_QUOTE_SPACE = re.compile(r'\s+([""\'\]}])')
_CLOSE_QUOTE_WORD = re.compile(r'([""\'\]}])([^\s])')
_DASH_NORMALIZE = re.compile(r'[—–]')
_QUOTE_NORMALIZE_DOUBLE = re.compile(r'[""]')
_QUOTE_NORMALIZE_SINGLE = re.compile(r'[\'\']')
_INVISIBLE_CHARS = re.compile(r'[\u200B-\u200D\uFEFF]')

def normalize_hindi_text(text):
    """
    Normalize Hindi text using regex patterns for proper formatting.
    
    This function:
    1. Normalizes whitespace
    2. Handles punctuation properly (no space before, one space after)
    3. Normalizes common characters (dashes, quotes)
    4. Removes invisible characters
    
    Args:
        text (str): Raw Hindi text
    
    Returns:
        str: Normalized Hindi text
    """
    if not text:
        return text
    
    # --- Step 1: Normalize multiple whitespaces to single space ---
    text = re.sub(r'\s+', ' ', text).strip()
    
    # --- Step 2: Handle punctuation properly ---
    
    # A. Remove any existing space *before* standard Hindi punctuation marks
    # (,, ;, :, !, ?, ।, ॥)
    text = re.sub(r'\s+([,;:!?।॥])', r'\1', text)

    # B. Ensure a single space *after* these punctuation marks
    # This correctly formats sentences like "है।अगला" to "है। अगला"
    text = re.sub(r'([,;:!?।॥])([^\s])', r'\1 \2', text)
    
    # C. Handle quotation marks and parentheses correctly
    # Hindi follows Western rules for these: no space inside the enclosure.
    
    # Remove space after opening quotes/parentheses
    text = re.sub(r'([“"‘\([{])\s+', r'\1', text)
    # Remove space before closing quotes/parentheses
    text = re.sub(r'\s+([”"’\])}])', r'\1', text)
    # Ensure a space *after* closing quotes/parentheses if a word follows
    text = re.sub(r'([”"’\])}])([^\s])', r'\1 \2', text)
    
    # --- Step 3: Normalize common characters ---

    # Normalize different types of dashes
    text = re.sub(r'[—–]', '-', text)
    
    # Normalize quotes (if you want to force straight quotes everywhere)
    text = re.sub(r'[“”]', '"', text)
    text = re.sub(r'[‘’]', "'", text)
    
    # --- Step 4: Remove zero-width characters and other invisible characters ---
    text = re.sub(r'[\u200B-\u200D\uFEFF]', '', text)  # Zero-width spaces etc.
    
    # Strip leading/trailing spaces again after final operations
    text = text.strip()
    
    return text

# Example Usage:

# raw_text = """
#     यह एक उदाहरण पाठ है,जो दिखाताहै कि स्पेसिंग कितनी ज़रूरीहै!
#     "यह एक उद्धरण है।" क्या यह काम करता है?
#     यह।सही है॥हाँ
# """

# normalized_text = normalize_hindi_text(raw_text)

# print("Original Text:")
# print(raw_text)
# print("\nNormalized Text:")
# print(normalized_text)

# Expected Output:
# Normalized Text:
# यह एक उदाहरण पाठ है, जो दिखाता है कि स्पेसिंग कितनी ज़रूरी है! "यह एक उद्धरण है।" क्या यह काम करता है? यह। सही है॥ हाँ


def separate_punctuation(text):
    """
    Separate punctuation marks from Hindi words using regex.
    
    This helps the tokenizer handle punctuation better by treating
    it as separate tokens.
    
    Args:
        text (str): Hindi text
    
    Returns:
        str: Text with punctuation separated
    """
    if not text:
        return text
    
    # Separate punctuation from Devanagari words
    # Pattern: word followed by punctuation or punctuation followed by word
    text = re.sub(r'([\u0900-\u097F\uA8E0-\uA8FF]+)([।॥,;:!?\-()\[\]{}"\'])', r'\1 \2', text)
    text = re.sub(r'([।॥,;:!?\-()\[\]{}"\'])([\u0900-\u097F\uA8E0-\uA8FF]+)', r'\1 \2', text)
    
    return text


def clean_hindi_text(text: str) -> str:
    """
    Comprehensive cleaning of Hindi text using regex.
    
    Combines normalization and punctuation separation.
    Optimized for large text processing.
    
    Args:
        text (str): Raw Hindi text
    
    Returns:
        str: Cleaned and normalized Hindi text
    """
    if not text:
        return text
    
    # Apply normalization (uses pre-compiled patterns)
    text = normalize_hindi_text(text)
    
    # Separate punctuation
    text = separate_punctuation(text)
    
    # Final whitespace normalization (use pre-compiled pattern)
    text = _WHITESPACE_PATTERN.sub(' ', text).strip()
    
    return text


def clean_hindi_text_streaming(file_path: str, chunk_size: int = 1024 * 1024) -> Iterator[str]:
    """
    Stream and clean large Hindi text files in chunks to avoid memory issues.
    
    This is optimized for very large files (>100MB) that don't fit in memory.
    
    Args:
        file_path (str): Path to the text file
        chunk_size (int): Size of chunks to read (default: 1MB)
    
    Yields:
        str: Cleaned text chunks
    """
    with open(file_path, 'r', encoding='utf-8') as f:
        buffer = ""
        while True:
            chunk = f.read(chunk_size)
            if not chunk:
                if buffer:
                    yield clean_hindi_text(buffer)
                break
            
            buffer += chunk
            
            # Process complete lines to avoid breaking in the middle of text
            while '\n' in buffer:
                line, buffer = buffer.split('\n', 1)
                if line.strip():
                    cleaned = clean_hindi_text(line)
                    if cleaned:
                        yield cleaned


def extract_hindi_words(text):
    """
    Extract only Hindi words (Devanagari script) from text using regex.
    
    Useful for filtering out non-Hindi content.
    
    Args:
        text (str): Mixed text
    
    Returns:
        list: List of Hindi words found
    """
    if not text:
        return []
    
    # Find all Devanagari words
    hindi_words = re.findall(DEVANAGARI_PATTERN, text)
    return hindi_words


def is_hindi_text(text):
    """
    Check if text contains Hindi (Devanagari script) using regex.
    
    Args:
        text (str): Text to check
    
    Returns:
        bool: True if text contains Devanagari characters
    """
    if not text:
        return False
    
    return bool(re.search(DEVANAGARI_PATTERN, text))


def filter_hindi_only(text: str, min_hindi_ratio: float = 0.7) -> str:
    """
    Filter text to keep only lines/sentences with significant Hindi content.
    
    This ensures the tokenizer only learns from Hindi text, not mixed content.
    Removes non-Hindi characters and keeps only Devanagari script with allowed punctuation.
    
    Args:
        text (str): Input text (may contain mixed Hindi/English/other)
        min_hindi_ratio (float): Minimum ratio of Devanagari chars to keep a line (0.0-1.0)
                                Default 0.7 means at least 70% Hindi characters
    
    Returns:
        str: Filtered text containing only Hindi-dominant lines
    """
    if not text:
        return text
    
    lines = text.split('\n')
    filtered_lines = []
    
    for line in lines:
        line = line.strip()
        if not line:
            continue
        
        # Count Devanagari characters
        devanagari_chars = len(re.findall(DEVANAGARI_PATTERN, line))
        total_chars = len(re.sub(r'\s', '', line))  # Exclude whitespace
        
        if total_chars == 0:
            continue
        
        # Calculate ratio of Hindi characters
        hindi_ratio = devanagari_chars / total_chars if total_chars > 0 else 0
        
        # Keep line if it has sufficient Hindi content
        if hindi_ratio >= min_hindi_ratio:
            # Extract only Devanagari words and allowed punctuation
            # Keep Hindi words, Hindi punctuation (।॥), and basic punctuation
            hindi_line = re.sub(
                r'[^\u0900-\u097F\uA8E0-\uA8FF\s।॥,;:!?\-()\[\]{}"\']+',
                ' ',
                line
            )
            # Normalize whitespace
            hindi_line = _WHITESPACE_PATTERN.sub(' ', hindi_line).strip()
            if hindi_line:
                filtered_lines.append(hindi_line)
    
    return '\n'.join(filtered_lines)

	# ============================================================================
	# HINDI TEXT PREPROCESSOR (OPTIMIZED FOR PYTHON 3.11)
	# ============================================================================
	# This module contains regex-based preprocessing functions for Hindi text
	# Handles Devanagari script patterns, punctuation, and normalization
	# Optimized for large datasets with compiled regex patterns

	import re
	from typing import Iterator, Optional


	# Devanagari Unicode ranges
	# Devanagari: U+0900 to U+097F
	# Devanagari Extended: U+A8E0 to U+A8FF
	DEVANAGARI_PATTERN = r'[\u0900-\u097F\uA8E0-\uA8FF]+'

	# Common Hindi punctuation marks
	HINDI_PUNCTUATION = r'[।॥,;:!?\-—–()\[\]{}"\'\'""]'

	# Numbers (both Devanagari and Arabic)
	DEVANAGARI_NUMBERS = r'[\u0966-\u096F]' # ०-९
	ARABIC_NUMBERS = r'[0-9]'

	# Pre-compile regex patterns for better performance (Python 3.11 optimization)
	# Compiling once and reusing is much faster than compiling on each call
	_WHITESPACE_PATTERN = re.compile(r'\s+')
	_SPACE_BEFORE_PUNCT = re.compile(r'\s+([,;:!?।॥])')
	_PUNCT_AFTER_SPACE = re.compile(r'([,;:!?।॥])([^\s])')
	_OPEN_QUOTE_SPACE = re.compile(r'([""\'\([{])\s+')
	_CLOSE_QUOTE_SPACE = re.compile(r'\s+([""\'\]}])')
	_CLOSE_QUOTE_WORD = re.compile(r'([""\'\]}])([^\s])')
	_DASH_NORMALIZE = re.compile(r'[—–]')
	_QUOTE_NORMALIZE_DOUBLE = re.compile(r'[""]')
	_QUOTE_NORMALIZE_SINGLE = re.compile(r'[\'\']')
	_INVISIBLE_CHARS = re.compile(r'[\u200B-\u200D\uFEFF]')

	def normalize_hindi_text(text):
	"""
	Normalize Hindi text using regex patterns for proper formatting.

	This function:
	1. Normalizes whitespace
	2. Handles punctuation properly (no space before, one space after)
	3. Normalizes common characters (dashes, quotes)
	4. Removes invisible characters

	Args:
	text (str): Raw Hindi text

	Returns:
	str: Normalized Hindi text
	"""
	if not text:
	return text

	# --- Step 1: Normalize multiple whitespaces to single space ---
	text = re.sub(r'\s+', ' ', text).strip()

	# --- Step 2: Handle punctuation properly ---

	# A. Remove any existing space before standard Hindi punctuation marks
	# (,, ;, :, !, ?, ।, ॥)
	text = re.sub(r'\s+([,;:!?।॥])', r'\1', text)

	# B. Ensure a single space after these punctuation marks
	# This correctly formats sentences like "है।अगला" to "है। अगला"
	text = re.sub(r'([,;:!?।॥])([^\s])', r'\1 \2', text)

	# C. Handle quotation marks and parentheses correctly
	# Hindi follows Western rules for these: no space inside the enclosure.

	# Remove space after opening quotes/parentheses
	text = re.sub(r'([“"‘\([{])\s+', r'\1', text)
	# Remove space before closing quotes/parentheses
	text = re.sub(r'\s+([”"’\])}])', r'\1', text)
	# Ensure a space after closing quotes/parentheses if a word follows
	text = re.sub(r'([”"’\])}])([^\s])', r'\1 \2', text)

	# --- Step 3: Normalize common characters ---

	# Normalize different types of dashes
	text = re.sub(r'[—–]', '-', text)

	# Normalize quotes (if you want to force straight quotes everywhere)
	text = re.sub(r'[“”]', '"', text)
	text = re.sub(r'[‘’]', "'", text)

	# --- Step 4: Remove zero-width characters and other invisible characters ---
	text = re.sub(r'[\u200B-\u200D\uFEFF]', '', text) # Zero-width spaces etc.

	# Strip leading/trailing spaces again after final operations
	text = text.strip()

	return text

	# Example Usage:

	# raw_text = """
	# यह एक उदाहरण पाठ है,जो दिखाताहै कि स्पेसिंग कितनी ज़रूरीहै!
	# "यह एक उद्धरण है।" क्या यह काम करता है?
	# यह।सही है॥हाँ
	# """

	# normalized_text = normalize_hindi_text(raw_text)

	# print("Original Text:")
	# print(raw_text)
	# print("\nNormalized Text:")
	# print(normalized_text)

	# Expected Output:
	# Normalized Text:
	# यह एक उदाहरण पाठ है, जो दिखाता है कि स्पेसिंग कितनी ज़रूरी है! "यह एक उद्धरण है।" क्या यह काम करता है? यह। सही है॥ हाँ


	def separate_punctuation(text):
	"""
	Separate punctuation marks from Hindi words using regex.

	This helps the tokenizer handle punctuation better by treating
	it as separate tokens.

	Args:
	text (str): Hindi text

	Returns:
	str: Text with punctuation separated
	"""
	if not text:
	return text

	# Separate punctuation from Devanagari words
	# Pattern: word followed by punctuation or punctuation followed by word
	text = re.sub(r'([\u0900-\u097F\uA8E0-\uA8FF]+)([।॥,;:!?\-()\[\]{}"\'])', r'\1 \2', text)
	text = re.sub(r'([।॥,;:!?\-()\[\]{}"\'])([\u0900-\u097F\uA8E0-\uA8FF]+)', r'\1 \2', text)

	return text


	def clean_hindi_text(text: str) -> str:
	"""
	Comprehensive cleaning of Hindi text using regex.

	Combines normalization and punctuation separation.
	Optimized for large text processing.

	Args:
	text (str): Raw Hindi text

	Returns:
	str: Cleaned and normalized Hindi text
	"""
	if not text:
	return text

	# Apply normalization (uses pre-compiled patterns)
	text = normalize_hindi_text(text)

	# Separate punctuation
	text = separate_punctuation(text)

	# Final whitespace normalization (use pre-compiled pattern)
	text = _WHITESPACE_PATTERN.sub(' ', text).strip()

	return text


	def clean_hindi_text_streaming(file_path: str, chunk_size: int = 1024 * 1024) -> Iterator[str]:
	"""
	Stream and clean large Hindi text files in chunks to avoid memory issues.

	This is optimized for very large files (>100MB) that don't fit in memory.

	Args:
	file_path (str): Path to the text file
	chunk_size (int): Size of chunks to read (default: 1MB)

	Yields:
	str: Cleaned text chunks
	"""
	with open(file_path, 'r', encoding='utf-8') as f:
	buffer = ""
	while True:
	chunk = f.read(chunk_size)
	if not chunk:
	if buffer:
	yield clean_hindi_text(buffer)
	break

	buffer += chunk

	# Process complete lines to avoid breaking in the middle of text
	while '\n' in buffer:
	line, buffer = buffer.split('\n', 1)
	if line.strip():
	cleaned = clean_hindi_text(line)
	if cleaned:
	yield cleaned


	def extract_hindi_words(text):
	"""
	Extract only Hindi words (Devanagari script) from text using regex.

	Useful for filtering out non-Hindi content.

	Args:
	text (str): Mixed text

	Returns:
	list: List of Hindi words found
	"""
	if not text:
	return []

	# Find all Devanagari words
	hindi_words = re.findall(DEVANAGARI_PATTERN, text)
	return hindi_words


	def is_hindi_text(text):
	"""
	Check if text contains Hindi (Devanagari script) using regex.

	Args:
	text (str): Text to check

	Returns:
	bool: True if text contains Devanagari characters
	"""
	if not text:
	return False

	return bool(re.search(DEVANAGARI_PATTERN, text))


	def filter_hindi_only(text: str, min_hindi_ratio: float = 0.7) -> str:
	"""
	Filter text to keep only lines/sentences with significant Hindi content.

	This ensures the tokenizer only learns from Hindi text, not mixed content.
	Removes non-Hindi characters and keeps only Devanagari script with allowed punctuation.

	Args:
	text (str): Input text (may contain mixed Hindi/English/other)
	min_hindi_ratio (float): Minimum ratio of Devanagari chars to keep a line (0.0-1.0)
	Default 0.7 means at least 70% Hindi characters

	Returns:
	str: Filtered text containing only Hindi-dominant lines
	"""
	if not text:
	return text

	lines = text.split('\n')
	filtered_lines = []

	for line in lines:
	line = line.strip()
	if not line:
	continue

	# Count Devanagari characters
	devanagari_chars = len(re.findall(DEVANAGARI_PATTERN, line))
	total_chars = len(re.sub(r'\s', '', line)) # Exclude whitespace

	if total_chars == 0:
	continue

	# Calculate ratio of Hindi characters
	hindi_ratio = devanagari_chars / total_chars if total_chars > 0 else 0

	# Keep line if it has sufficient Hindi content
	if hindi_ratio >= min_hindi_ratio:
	# Extract only Devanagari words and allowed punctuation
	# Keep Hindi words, Hindi punctuation (।॥), and basic punctuation
	hindi_line = re.sub(
	r'[^\u0900-\u097F\uA8E0-\uA8FF\s।॥,;:!?\-()\[\]{}"\']+',
	' ',
	line
	)
	# Normalize whitespace
	hindi_line = _WHITESPACE_PATTERN.sub(' ', hindi_line).strip()
	if hindi_line:
	filtered_lines.append(hindi_line)

	return '\n'.join(filtered_lines)