# scripts/preprocess.py
import re

def clean_text(text):
    text = re.sub(r'[^\u0621-\u064A\s]', '', text)  # Arabic letters only
    text = re.sub(r'\s+', ' ', text).strip()
    return text

if __name__ == "__main__":
    with open('../data/raw_data/iraqi_data.txt', 'r') as f:
        raw_data = f.readlines()
    
    cleaned_data = [clean_text(line) for line in raw_data]
    with open('../data/cleaned_data/iraqi_data_cleaned.txt', 'w') as f:
        for line in cleaned_data:
            f.write(line + '\n')