File size: 516 Bytes
ea3d866 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 | # scripts/preprocess.py
import re
def clean_text(text):
text = re.sub(r'[^\u0621-\u064A\s]', '', text) # Arabic letters only
text = re.sub(r'\s+', ' ', text).strip()
return text
if __name__ == "__main__":
with open('../data/raw_data/iraqi_data.txt', 'r') as f:
raw_data = f.readlines()
cleaned_data = [clean_text(line) for line in raw_data]
with open('../data/cleaned_data/iraqi_data_cleaned.txt', 'w') as f:
for line in cleaned_data:
f.write(line + '\n')
|