Spaces:

al6ert
/

nodding

Runtime error

Al6ert

draw data

85a54f3 over 3 years ago

4.56 kB

import os
from datetime import datetime, timedelta
import ccxt
import pandas as pd
#import numpy as np
import calendar
#from scipy import signal
from scipy.signal import savgol_filter
from pykalman import KalmanFilter


class HistoricalDataFetcher:

  def __init__(self,
               exchange_name='coinbasepro',
               symbol='BTC/USD',
               timeframe='1d',
               start_date=datetime(2015, 7, 1),
               end_date=datetime.now()):
    self.exchange = getattr(ccxt, exchange_name)()
    self.symbol = symbol
    self.timeframe = timeframe
    self.start_date = start_date
    self.end_date = end_date
    self.raw_path = f"data/{self.symbol.replace('/','')}"
    self.clean_path = f"data/clean_{self.symbol.replace('/','')}"

    # Crear la carpeta para los archivos raw si no existe
    if not os.path.exists(self.raw_path):
      os.makedirs(self.raw_path)
    # Crear la carpeta para los archivos limpios si no existe
    if not os.path.exists(self.clean_path):
      os.makedirs(self.clean_path)

  def fetch_data(self):
    start_date = self.start_date
    end_date = self.end_date

    while start_date < end_date:

      year = start_date.year
      month = start_date.month

      # Determine the number of days in the month
      num_days = calendar.monthrange(year, month)[1]

      # Calculate the start and end dates for the month
      since = self.exchange.parse8601(
        start_date.strftime('%Y-%m-%d') + 'T00:00:00Z')
      until = self.exchange.parse8601(
        (start_date + timedelta(days=num_days)).strftime('%Y-%m-%d') +
        'T23:59:59Z')

      print(f"Fetching data for {calendar.month_name[month]} {year}")

      # Get historical data for the month
      ohlcv = self.exchange.fetch_ohlcv(self.symbol,
                                        self.timeframe,
                                        since,
                                        limit=None,
                                        params={'end': until})

      # Convert the data to a pandas DataFrame
      df = pd.DataFrame(
        ohlcv, columns=['timestamp', 'open', 'high', 'low', 'close', 'volume'])

      # Convert the timestamp to a datetime and set it as the index
      df['timestamp'] = pd.to_datetime(df['timestamp'], unit='ms')
      df.set_index('timestamp', inplace=True)

      # Save the data to a CSV file
      filename = f"{self.symbol.replace('/','')}_{self.timeframe}_{start_date.strftime('%Y%m')}.csv"
      file_path = os.path.join(self.raw_path, filename)

      df.to_csv(file_path)

      print(f"Data saved to {filename}")

      # Increment the start date to the next month
      start_date = start_date.replace(day=1) + timedelta(days=32)
      start_date = start_date.replace(day=1)

  def get_data(self):
    # Obtener una lista de todos los archivos en la carpeta "data"
    data_files = os.listdir(self.raw_path)
    # Crear una lista vacía para almacenar los DataFrames de cada archivo
    data_frames = []

    # Cargar cada archivo CSV y agregar su DataFrame a la lista
    for file_name in data_files:
      if file_name.endswith(".csv"):
        file_path = os.path.join(self.raw_path, file_name)
        data = pd.read_csv(file_path, index_col=0, parse_dates=True)
        data_frames.append(data)

    # Concatenar los DataFrames en un solo DataFrame
    all_data = pd.concat(data_frames)
    return all_data

  def apply_filters(self,
                    data,
                    window_size=11,
                    polyorder=2,
                    kalman_observation_covariance=0.003,
                    kalman_transition_covariance=0.01,
                    kalman_initial_state_covariance=1000):

    # Aplicar suavizado exponencial
    alpha = 0.5
    data['close_ewm'] = data['close'].ewm(alpha=alpha, adjust=False).mean()

    # Aplicar filtro de Kalman
    kf = KalmanFilter(observation_covariance=kalman_observation_covariance,
                      transition_covariance=kalman_transition_covariance,
                      initial_state_covariance=kalman_initial_state_covariance,
                      n_dim_obs=1)
    data['close_kf'], _ = kf.filter(data['close'].values)

    # Aplicar suavizado de Savitzky-Golay
    data['close_savgol'] = savgol_filter(data['close'], window_size, polyorder)

    # Guardar los datos suavizados en un archivo CSV
    filename = f"{self.symbol.replace('/','')}_{self.timeframe}_{self.start_date.strftime('%Y%m')}_cleaned.csv"
    file_path = os.path.join(self.clean_path, filename)

    data.to_csv(file_path)

    return data

	import os
	from datetime import datetime, timedelta
	import ccxt
	import pandas as pd
	#import numpy as np
	import calendar
	#from scipy import signal
	from scipy.signal import savgol_filter
	from pykalman import KalmanFilter


	class HistoricalDataFetcher:

	def __init__(self,
	exchange_name='coinbasepro',
	symbol='BTC/USD',
	timeframe='1d',
	start_date=datetime(2015, 7, 1),
	end_date=datetime.now()):
	self.exchange = getattr(ccxt, exchange_name)()
	self.symbol = symbol
	self.timeframe = timeframe
	self.start_date = start_date
	self.end_date = end_date
	self.raw_path = f"data/{self.symbol.replace('/','')}"
	self.clean_path = f"data/clean_{self.symbol.replace('/','')}"

	# Crear la carpeta para los archivos raw si no existe
	if not os.path.exists(self.raw_path):
	os.makedirs(self.raw_path)
	# Crear la carpeta para los archivos limpios si no existe
	if not os.path.exists(self.clean_path):
	os.makedirs(self.clean_path)

	def fetch_data(self):
	start_date = self.start_date
	end_date = self.end_date

	while start_date < end_date:

	year = start_date.year
	month = start_date.month

	# Determine the number of days in the month
	num_days = calendar.monthrange(year, month)[1]

	# Calculate the start and end dates for the month
	since = self.exchange.parse8601(
	start_date.strftime('%Y-%m-%d') + 'T00:00:00Z')
	until = self.exchange.parse8601(
	(start_date + timedelta(days=num_days)).strftime('%Y-%m-%d') +
	'T23:59:59Z')

	print(f"Fetching data for {calendar.month_name[month]} {year}")

	# Get historical data for the month
	ohlcv = self.exchange.fetch_ohlcv(self.symbol,
	self.timeframe,
	since,
	limit=None,
	params={'end': until})

	# Convert the data to a pandas DataFrame
	df = pd.DataFrame(
	ohlcv, columns=['timestamp', 'open', 'high', 'low', 'close', 'volume'])

	# Convert the timestamp to a datetime and set it as the index
	df['timestamp'] = pd.to_datetime(df['timestamp'], unit='ms')
	df.set_index('timestamp', inplace=True)

	# Save the data to a CSV file
	filename = f"{self.symbol.replace('/','')}_{self.timeframe}_{start_date.strftime('%Y%m')}.csv"
	file_path = os.path.join(self.raw_path, filename)

	df.to_csv(file_path)

	print(f"Data saved to {filename}")

	# Increment the start date to the next month
	start_date = start_date.replace(day=1) + timedelta(days=32)
	start_date = start_date.replace(day=1)

	def get_data(self):
	# Obtener una lista de todos los archivos en la carpeta "data"
	data_files = os.listdir(self.raw_path)
	# Crear una lista vacía para almacenar los DataFrames de cada archivo
	data_frames = []

	# Cargar cada archivo CSV y agregar su DataFrame a la lista
	for file_name in data_files:
	if file_name.endswith(".csv"):
	file_path = os.path.join(self.raw_path, file_name)
	data = pd.read_csv(file_path, index_col=0, parse_dates=True)
	data_frames.append(data)

	# Concatenar los DataFrames en un solo DataFrame
	all_data = pd.concat(data_frames)
	return all_data

	def apply_filters(self,
	data,
	window_size=11,
	polyorder=2,
	kalman_observation_covariance=0.003,
	kalman_transition_covariance=0.01,
	kalman_initial_state_covariance=1000):

	# Aplicar suavizado exponencial
	alpha = 0.5
	data['close_ewm'] = data['close'].ewm(alpha=alpha, adjust=False).mean()

	# Aplicar filtro de Kalman
	kf = KalmanFilter(observation_covariance=kalman_observation_covariance,
	transition_covariance=kalman_transition_covariance,
	initial_state_covariance=kalman_initial_state_covariance,
	n_dim_obs=1)
	data['close_kf'], _ = kf.filter(data['close'].values)

	# Aplicar suavizado de Savitzky-Golay
	data['close_savgol'] = savgol_filter(data['close'], window_size, polyorder)

	# Guardar los datos suavizados en un archivo CSV
	filename = f"{self.symbol.replace('/','')}_{self.timeframe}_{self.start_date.strftime('%Y%m')}_cleaned.csv"
	file_path = os.path.join(self.clean_path, filename)

	data.to_csv(file_path)

	return data