# scripts/scrape_data.py
import requests
from bs4 import BeautifulSoup
import os

def scrape_iraqi_arabic_text(url):
    response = requests.get(url)
    response.raise_for_status()  # Check if the request was successful

    soup = BeautifulSoup(response.content, "html.parser")

    # Update these to match the actual tag and class on the website
    proverbs = soup.find_all("p", class_="proverb-text")  # Replace with the correct tag and class

    iraqi_text = []
    for proverb in proverbs:
        text = proverb.get_text().strip()
        if text:  # Only add non-empty lines
            iraqi_text.append(text)

    if not iraqi_text:
        print("No proverbs found. Check the tag and class names in the code.")
    return iraqi_text

def save_text_to_file(text_list, relative_filepath):
    project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
    absolute_filepath = os.path.join(project_root, relative_filepath)
    os.makedirs(os.path.dirname(absolute_filepath), exist_ok=True)

    with open(absolute_filepath, 'w', encoding='utf-8') as f:
        for line in text_list:
            f.write(line + '\n')
    print(f"Data saved to {absolute_filepath}")

if __name__ == "__main__":
    url = "https://ar.mo3jam.com/dialect/Iraqi"
    iraqi_data = scrape_iraqi_arabic_text(url)
    
    # Check if any data was scraped
    if iraqi_data:
        save_text_to_file(iraqi_data, "data/raw_data/iraqi_data.txt")
    else:
        print("No data found. Please check the scraping criteria.")