scripts/preprocess_data.py

import pandas as pd

def preprocess_data(input_file, output_file):
    data = []
    with open(input_file, "r", encoding="utf-8") as f:
        for line in f:
            if " - " in line:
                proverb, meaning = line.split(" - ", 1)
                data.append({"proverb": proverb.strip(), "meaning": meaning.strip()})

    # Save as a DataFrame
    df = pd.DataFrame(data)
    df.to_csv(output_file, index=False)
    print(f"Data saved to {output_file}")

# Example usage
input_file = "data/raw_data/iraqi_data.txt"
output_file = "data/processed_data.csv"
preprocess_data(input_file, output_file)