| # scripts/preprocess.py | |
| import re | |
| def clean_text(text): | |
| text = re.sub(r'[^\u0621-\u064A\s]', '', text) # Arabic letters only | |
| text = re.sub(r'\s+', ' ', text).strip() | |
| return text | |
| if __name__ == "__main__": | |
| with open('../data/raw_data/iraqi_data.txt', 'r') as f: | |
| raw_data = f.readlines() | |
| cleaned_data = [clean_text(line) for line in raw_data] | |
| with open('../data/cleaned_data/iraqi_data_cleaned.txt', 'w') as f: | |
| for line in cleaned_data: | |
| f.write(line + '\n') | |