chunking.py

import re
from transformers import AutoTokenizer, AutoModelForSeq2SeqLM

def get_max_word_length(target_languages: list[str]) -> int:
    helsinki_word_limits = {
        "el": 50,
        "et": 50,
        "fi": 50,
        "fr": 40,
        "sv": 140,
        "hu": 50,
        "lt": 50,
        "sk": 140,
        "bg": 50,
        "cs": 140,
        "da": 140,
        "de": 150,
        "pl": 120, # New Languages Start Here Need To Test
        "mk": 20,
        "es": 50,
        "it": 60,
        "is": 90,
        "tr": 60,
        "no": 50,
        "lb": 140,
        "sq": 140,
        "sl": 140,
        "cnr": 60,
        "nl": 50,
        "pt": 50,
        "lv": 50,
        "ro": 50,
    }

    max_word_length = 700  # Default Chunk Size

    for lang in target_languages:
        if lang in helsinki_word_limits:
            if helsinki_word_limits[lang] < max_word_length:
                max_word_length = helsinki_word_limits[lang]

    return max_word_length

def chunk_text(text: str, safe_word_limit: int) -> list[str]:
    sentences = re.split(r'(?<=[.!?])\s+', text.strip())
    
    chunks = []
    current_chunk = []
    current_word_count = 0

    for sentence in sentences:
        sentence = sentence.strip()
        if not sentence:
            continue

        word_count = len(sentence.split())

        # If sentence is longer than the safe word limit by itself, split it
        if word_count > safe_word_limit:
            if current_chunk:
                chunks.append(' '.join(current_chunk))
                current_chunk = []
                current_word_count = 0
            words = sentence.split()
            for i in range(0, len(words), safe_word_limit):
                chunks.append(' '.join(words[i:i+safe_word_limit]))
            continue

        # Otherwise, see if it fits in the current chunk
        if current_word_count + word_count <= safe_word_limit:
            current_chunk.append(sentence)
            current_word_count += word_count
        else:
            # Start a new chunk
            chunks.append(' '.join(current_chunk))
            current_chunk = [sentence]
            current_word_count = word_count

    if current_chunk:
        chunks.append(' '.join(current_chunk))

    return chunks