| # scripts/scrape_data.py | |
| import requests | |
| from bs4 import BeautifulSoup | |
| import os | |
| def scrape_iraqi_arabic_text(url): | |
| response = requests.get(url) | |
| response.raise_for_status() # Check if the request was successful | |
| soup = BeautifulSoup(response.content, "html.parser") | |
| # Update these to match the actual tag and class on the website | |
| proverbs = soup.find_all("p", class_="proverb-text") # Replace with the correct tag and class | |
| iraqi_text = [] | |
| for proverb in proverbs: | |
| text = proverb.get_text().strip() | |
| if text: # Only add non-empty lines | |
| iraqi_text.append(text) | |
| if not iraqi_text: | |
| print("No proverbs found. Check the tag and class names in the code.") | |
| return iraqi_text | |
| def save_text_to_file(text_list, relative_filepath): | |
| project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) | |
| absolute_filepath = os.path.join(project_root, relative_filepath) | |
| os.makedirs(os.path.dirname(absolute_filepath), exist_ok=True) | |
| with open(absolute_filepath, 'w', encoding='utf-8') as f: | |
| for line in text_list: | |
| f.write(line + '\n') | |
| print(f"Data saved to {absolute_filepath}") | |
| if __name__ == "__main__": | |
| url = "https://ar.mo3jam.com/dialect/Iraqi" | |
| iraqi_data = scrape_iraqi_arabic_text(url) | |
| # Check if any data was scraped | |
| if iraqi_data: | |
| save_text_to_file(iraqi_data, "data/raw_data/iraqi_data.txt") | |
| else: | |
| print("No data found. Please check the scraping criteria.") | |