| """
|
| Data loader module for BTC 1-minute OHLCV data.
|
|
|
| This module handles loading, validation, and preprocessing of historical
|
| Bitcoin price data from CSV files.
|
| """
|
|
|
| import pandas as pd
|
| import os
|
| from typing import Optional
|
|
|
|
|
| def load_data(
|
| filepath: str,
|
| start_date: Optional[str] = None,
|
| end_date: Optional[str] = None,
|
| validate: bool = True
|
| ) -> pd.DataFrame:
|
| """
|
| Load BTC 1-minute OHLCV data from CSV file.
|
|
|
| Parameters
|
| ----------
|
| filepath : str
|
| Path to the CSV file containing OHLCV data
|
| start_date : str, optional
|
| Start date for filtering (format: 'YYYY-MM-DD')
|
| end_date : str, optional
|
| End date for filtering (format: 'YYYY-MM-DD')
|
| validate : bool, default True
|
| Whether to validate the data integrity
|
|
|
| Returns
|
| -------
|
| pd.DataFrame
|
| DataFrame with columns: timestamp, open, high, low, close, volume
|
| Indexed by timestamp
|
|
|
| Raises
|
| ------
|
| FileNotFoundError
|
| If the file doesn't exist
|
| ValueError
|
| If required columns are missing or data is invalid
|
| """
|
| if not os.path.exists(filepath):
|
| raise FileNotFoundError(f"Data file not found: {filepath}")
|
|
|
|
|
| print(f"Loading data from {filepath}...")
|
| df = pd.read_csv(filepath)
|
|
|
|
|
| required_cols = ['timestamp', 'open', 'high', 'low', 'close', 'volume']
|
| missing_cols = [col for col in required_cols if col not in df.columns]
|
| if missing_cols:
|
| raise ValueError(f"Missing required columns: {missing_cols}")
|
|
|
|
|
| df['timestamp'] = pd.to_datetime(df['timestamp'])
|
|
|
|
|
| df = df.sort_values('timestamp').reset_index(drop=True)
|
|
|
|
|
| if start_date:
|
| df = df[df['timestamp'] >= pd.to_datetime(start_date)]
|
| if end_date:
|
| df = df[df['timestamp'] <= pd.to_datetime(end_date)]
|
|
|
|
|
| df = df.set_index('timestamp')
|
|
|
| if validate:
|
| _validate_data(df)
|
|
|
| print(f"✓ Loaded {len(df)} bars from {df.index[0]} to {df.index[-1]}")
|
|
|
| return df
|
|
|
|
|
| def _validate_data(df: pd.DataFrame) -> None:
|
| """
|
| Validate data integrity.
|
|
|
| Parameters
|
| ----------
|
| df : pd.DataFrame
|
| DataFrame to validate
|
|
|
| Raises
|
| ------
|
| ValueError
|
| If data validation fails
|
| """
|
|
|
| if df.isnull().any().any():
|
| null_counts = df.isnull().sum()
|
| print(f"⚠ Warning: Found missing values:\n{null_counts[null_counts > 0]}")
|
|
|
| df.fillna(method='ffill', inplace=True)
|
| print(" → Filled missing values using forward fill")
|
|
|
|
|
| tolerance = 0.01
|
| invalid_high = (df['high'] < df['low'] - tolerance).sum()
|
| invalid_open = ((df['open'] > df['high'] + tolerance) | (df['open'] < df['low'] - tolerance)).sum()
|
| invalid_close = ((df['close'] > df['high'] + tolerance) | (df['close'] < df['low'] - tolerance)).sum()
|
|
|
| if invalid_high > 0:
|
| print(f"⚠ Warning: Found {invalid_high} bars where high < low (fixing automatically)")
|
|
|
| mask = df['high'] < df['low']
|
| df.loc[mask, ['high', 'low']] = df.loc[mask, ['low', 'high']].values
|
| if invalid_open > 0:
|
| print(f"⚠ Warning: Found {invalid_open} bars where open is outside [low, high] (fixing automatically)")
|
|
|
| df['open'] = df['open'].clip(lower=df['low'], upper=df['high'])
|
| if invalid_close > 0:
|
| print(f"⚠ Warning: Found {invalid_close} bars where close is outside [low, high] (fixing automatically)")
|
|
|
| df['close'] = df['close'].clip(lower=df['low'], upper=df['high'])
|
|
|
|
|
| if (df[['open', 'high', 'low', 'close', 'volume']] < 0).any().any():
|
| raise ValueError("Found negative prices or volume")
|
|
|
|
|
| if (df[['open', 'high', 'low', 'close']] == 0).any().any():
|
| raise ValueError("Found zero prices")
|
|
|
| print("✓ Data validation passed")
|
|
|
|
|
| def get_data_summary(df: pd.DataFrame) -> dict:
|
| """
|
| Get summary statistics for the dataset.
|
|
|
| Parameters
|
| ----------
|
| df : pd.DataFrame
|
| OHLCV DataFrame
|
|
|
| Returns
|
| -------
|
| dict
|
| Dictionary containing summary statistics
|
| """
|
| summary = {
|
| 'num_bars': len(df),
|
| 'start_date': df.index[0],
|
| 'end_date': df.index[-1],
|
| 'duration_days': (df.index[-1] - df.index[0]).days,
|
| 'price_range': {
|
| 'min': df['low'].min(),
|
| 'max': df['high'].max(),
|
| 'mean': df['close'].mean(),
|
| 'std': df['close'].std()
|
| },
|
| 'volume': {
|
| 'total': df['volume'].sum(),
|
| 'mean': df['volume'].mean(),
|
| 'median': df['volume'].median()
|
| }
|
| }
|
| return summary
|
|
|
|
|
| if __name__ == "__main__":
|
|
|
| import sys
|
| sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
| import config
|
|
|
| try:
|
| df = load_data(config.BTC_DATA_FILE)
|
| summary = get_data_summary(df)
|
|
|
| print("\nData Summary:")
|
| print(f" Bars: {summary['num_bars']:,}")
|
| print(f" Period: {summary['start_date']} to {summary['end_date']}")
|
| print(f" Duration: {summary['duration_days']} days")
|
| print(f" Price range: ${summary['price_range']['min']:.2f} - ${summary['price_range']['max']:.2f}")
|
| print(f" Mean price: ${summary['price_range']['mean']:.2f}")
|
|
|
| except FileNotFoundError:
|
| print("⚠ Data file not found. Please add btc_1m.csv to the data/ directory.")
|
| except Exception as e:
|
| print(f"✗ Error: {e}")
|
|
|
