File size: 1,522 Bytes
ea3d866 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 | # scripts/scrape_data.py
import requests
from bs4 import BeautifulSoup
import os
def scrape_iraqi_arabic_text(url):
response = requests.get(url)
response.raise_for_status() # Check if the request was successful
soup = BeautifulSoup(response.content, "html.parser")
# Update these to match the actual tag and class on the website
proverbs = soup.find_all("p", class_="proverb-text") # Replace with the correct tag and class
iraqi_text = []
for proverb in proverbs:
text = proverb.get_text().strip()
if text: # Only add non-empty lines
iraqi_text.append(text)
if not iraqi_text:
print("No proverbs found. Check the tag and class names in the code.")
return iraqi_text
def save_text_to_file(text_list, relative_filepath):
project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
absolute_filepath = os.path.join(project_root, relative_filepath)
os.makedirs(os.path.dirname(absolute_filepath), exist_ok=True)
with open(absolute_filepath, 'w', encoding='utf-8') as f:
for line in text_list:
f.write(line + '\n')
print(f"Data saved to {absolute_filepath}")
if __name__ == "__main__":
url = "https://ar.mo3jam.com/dialect/Iraqi"
iraqi_data = scrape_iraqi_arabic_text(url)
# Check if any data was scraped
if iraqi_data:
save_text_to_file(iraqi_data, "data/raw_data/iraqi_data.txt")
else:
print("No data found. Please check the scraping criteria.")
|