Spaces:
Configuration error
Configuration error
File size: 3,941 Bytes
778eb4c fe3c27e 778eb4c bbd7911 778eb4c 314de7a 778eb4c 314de7a 778eb4c 314de7a 778eb4c fe3c27e 778eb4c b3c3c9a 314de7a 778eb4c 82cc063 fe3c27e 778eb4c f9f1148 778eb4c f9f1148 778eb4c fe3c27e 778eb4c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 | import os
import logging
import sys
from datetime import datetime
from pathlib import Path
from dotenv import load_dotenv
from huggingface_hub import HfApi
from zoneinfo import ZoneInfo
# Import modules
from ingestion.jobstreet.jobstreet_scraper import js_scraper, get_total_pages
from ingestion.linkedin.linkedin_scraper import ld_scraper
SEARCH_TERMS = [
"Data Engineer",
"Data Analyst",
"Data Scientist",
"Machine Learning Engineer",
]
js_date_range = {
"daily": 1,
"weekly": 7,
"montly": 31,
"None": None
}
ld_date_range = {
"daily": "r86400",
"weekly": "r604800",
"montly": "r2592000",
"None": None
}
app_logger = logging.getLogger(__name__)
def upload_to_hf():
"""Sends local JSONL data and active log files to permanent HF Dataset storage"""
token = os.environ.get("HF_TOKEN")
if not token:
app_logger.warning("⚠️ HF_TOKEN not found in environment variables. Skipping cloud sync.")
return
api = HfApi(token=token)
repo_target = "Amin1600/Web_Scraper_Data"
app_logger.info("📦 Commencing synchronization with Hugging Face Dataset vault...")
try:
# 1. Upload your data folder contents (.jsonl files)
if os.path.exists("data") and os.listdir("data"):
api.upload_folder(
folder_path="data",
path_in_repo="job_data", # Folder name inside your dataset repo
repo_id=repo_target,
repo_type="dataset"
)
app_logger.info("🎉 Scraping data (.jsonl) successfully backed up to HF Datasets!")
# 2. Upload your logs folder contents (.log files)
if os.path.exists("logs") and os.listdir("logs"):
api.upload_folder(
folder_path="logs",
path_in_repo="logs", # Folder name inside your dataset repo
repo_id=repo_target,
repo_type="dataset"
)
app_logger.info("🎉 System execution logs successfully backed up to HF Datasets!")
except Exception as e:
app_logger.error(f"❌ Failed to sync files to Hugging Face Dataset: {e}")
def job_scraper(job_title="Data Analyst", target_location="Kuala Lumpur", date_range="None", run_type="scheduled"):
kl_timezone = ZoneInfo("Asia/Kuala_Lumpur")
timestamp = datetime.now(kl_timezone).strftime('%d-%m-%Y, %H:%M:%S')
if date_range == "all" or date_range is None:
date_range = "None"
if run_type == 'scheduled':
app_logger.info("🚀 Scheduler successfully armed for midnight Malaysia Time.")
app_logger.info(f"⏰ Starting Scheduled web scraper run for {job_title} job listings. {timestamp}")
app_logger.info("🚀 Scraping jobs from Jobstreet")
js_scraper(job_type = job_title, location = target_location, date_range = js_date_range[date_range])
app_logger.info("🚀 Scraping jobs from Linkedin")
ld_scraper(job_type = job_title, location = target_location, date_range = ld_date_range[date_range])
app_logger.info("🏁 All scraping tasks completed successfully.")
upload_to_hf()
elif run_type == 'manual':
app_logger.info(f"Manual run, searching for {date_range or "all"} listings.")
app_logger.info(f"Starting web scraper run for {job_title} job listings. {timestamp}")
app_logger.info("🚀 Scraping jobs from Jobstreet")
js_scraper(job_type = job_title, location = target_location, date_range = js_date_range[date_range])
app_logger.info("🚀 Scraping jobs from Linkedin")
ld_scraper(job_type = job_title, location = target_location, date_range = ld_date_range[date_range])
app_logger.info("🏁 All scraping tasks completed successfully.")
upload_to_hf()
else:
app_logger.warning('🛑 Undetermined run type task is unable to start!')
|