SyedaArisha's picture
Upload folder using huggingface_hub
baf834b verified
Raw History Blame Contribute Delete
1.39 kB
"""
Shared utilities for preprocessing modules.
Provides logging setup, seed initialization, and a timing decorator
used by all dataset-specific preprocessors.
"""
import os
import time
import logging
import random
import numpy as np
import joblib
import pandas as pd
from sklearn.preprocessing import LabelEncoder, StandardScaler, MinMaxScaler
try:
from utils.project_paths import LOGS_DIR
except ImportError:
from project_paths import LOGS_DIR
# Setup logging configuration
os.makedirs(LOGS_DIR, exist_ok=True)
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s [%(levelname)s] %(message)s',
handlers=[
logging.StreamHandler(),
logging.FileHandler(os.path.join(LOGS_DIR, "preprocessing.log"))
]
)
logger = logging.getLogger(__name__)
def set_seeds(seed: int = 42):
"""Set random seeds for reproducibility."""
random.seed(seed)
np.random.seed(seed)
logger.info(f"Random seeds set to {seed} for reproducibility.")
def time_tracker(func):
"""Decorator to measure and log the execution time of functions."""
def wrapper(*args, **kwargs):
start_time = time.time()
logger.info(f"Starting: {func.__name__}")
result = func(*args, **kwargs)
elapsed = time.time() - start_time
logger.info(f"Finished: {func.__name__} ({elapsed:.2f}s)")
return result
return wrapper