version5

Loader.py

@@ -0,0 +1,73 @@
+import os
+from stable_baselines3 import PPO
+import gym
+
+
+
+
+class MegaLoader:
+    def __init__(self, env_name, algo, directory):
+        self.env_name = env_name
+        self.env = gym.make(self.env_name)
+        self.env.reset()
+
+        self.algorithm = algo
+        self.directory = directory
+
+    def model_selector(self, all_models_dir):
+        # print( os.listdir(all_models_dir)[-1])
+        return os.listdir(all_models_dir)[-1]
+
+        # return whole file name properly, with date and all
+        pass
+
+    def load(self, execution_number, no_of_episodes):
+        # Input :- 
+        #    1. execution_number - 
+        #    2. no_of_episodes - 
+        #    3. 
+
+        # Returns :-
+        #    Dictionary with the following items
+            #    1. total_return       [int]
+            #    2. return_per_episode [list]
+            #    3.
+            #    3.
+            #    3.
+            #    3.
+
+        total_return = 0
+        return_per_episode = []
+        info_per_episode = []
+
+        all_models_dir = f"model/{self.directory}/{execution_number}"
+
+        model_no = self.model_selector(all_models_dir)
+        model_dir = f"{all_models_dir}/{model_no}"
+
+        model = PPO.load(model_dir, env=self.env)
+
+        for ep in range(no_of_episodes):
+            obs = self.env.reset()
+            done = False
+            info = ""
+            episode_return = 0
+            
+            while not done:
+                action, _states = model.predict(obs)
+                obs, rewards, done, info = self.env.step(action)
+                episode_return += rewards
+
+                # self.env.render()
+
+            return_per_episode.append(episode_return)
+            total_return += episode_return
+            info_per_episode.append(info)
+
+        model_test_info = {
+            "total_return": total_return,
+            "return_per_episode": return_per_episode,
+            "info_per_episode": info_per_episode,
+        }
+
+        return model_test_info

PPO_HypConfig.py

@@ -0,0 +1,489 @@
+import random
+import optuna
+import numpy as np
+
+## change default to "selector" for that parameter
+## there are specifications for what type should be there, see that and do final design. FOr exxample --> learning_rate (Union[float, Callable[[float], float]])
+## Fix TENSORBOARD_LOG
+## Generate a number/code that is unique to each hyperparmeter configuration
+
+
+
+# import stable_baselines3.common.schedules as schedules
+
+
+
+#####################################################################
+##             LEARNING RATE SCHEDULER (learning_rate)             ##
+#####################################################################
+# Currently random, make it categorical and pass parameters using dictionary properly
+
+class Learning_Rate_Scheduler:
+    # remove this default function later
+    # Make this categorical
+    # send params in __init__ for all class's functions --> def __init__(self, i, params):
+    # always call through 
+    
+
+    def __init__(self):
+        self.low = 1e-5
+        self.high = 1e-2
+        self.log = True
+        #do this for all
+    
+    def default(self):
+        return random.random() / 1000
+
+    def opt(self, trial):
+        return trial.suggest_float("learning_rate", self.low, self.high, log=self.log)
+
+    def step_decay(optimizer, initial_lr=0.001, drop_rate=0.1, epochs_drop=10):
+        return
+        scheduler = lr_scheduler.StepLR(optimizer, step_size=epochs_drop, gamma=drop_rate)
+        return scheduler
+
+    def exp_decay(optimizer, initial_lr=0.001, decay_rate=0.96):
+        return
+        scheduler = lr_scheduler.ExponentialLR(optimizer, gamma=decay_rate)
+        return scheduler
+
+    def cosine_anneal(optimizer, initial_lr=0.001, epochs=100):
+        return
+        scheduler = lr_scheduler.CosineAnnealingLR(optimizer, T_max=epochs)
+        return scheduler
+
+    def reduce_on_plateau(optimizer, mode='min', factor=0.1, patience=10, verbose=False):
+        return
+        scheduler = lr_scheduler.ReduceLROnPlateau(optimizer, mode=mode, factor=factor, patience=patience, verbose=verbose)
+        return scheduler
+
+    def cyclical_lr(optimizer, step_size=2000, base_lr=1e-3, max_lr=6e-3, mode='triangular', gamma=1.):
+        return
+        scheduler = lr_scheduler.CyclicLR(optimizer, base_lr=base_lr, max_lr=max_lr, step_size_up=step_size,
+                                        mode=mode, gamma=gamma)
+        return scheduler
+
+    def warmup_lr(optimizer, factor=10, warmup_epochs=5):
+        return
+        def lr_lambda(current_epoch):
+            if current_epoch < warmup_epochs:
+                return factor
+            else:
+                return 1.0
+        scheduler = lr_scheduler.LambdaLR(optimizer, lr_lambda)
+        return scheduler
+
+    def one_cycle_lr(optimizer, num_steps, lr_range=(1e-4, 1e-2), momentum_range=(0.85, 0.95)):
+        return
+        scheduler = lr_scheduler.OneCycleLR(optimizer, max_lr=lr_range[1], total_steps=num_steps,
+                                            pct_start=0.3, anneal_strategy='cos', cycle_momentum=True,
+                                            base_momentum=momentum_range[0], max_momentum=momentum_range[1])
+        return scheduler
+
+    def constant_schedule(self, lr=3e-4):
+        return schedules.constant_schedule(3e-4)
+
+    def linear_schedule(self, initial_value=1e6, final_value=3e-4, n_timsteps=1e-4):
+        return schedules.linear_schedule(initial_value, final_value, n_timsteps)
+
+    def piecewise_schedule(self, schedule_pieces):
+        return schedules.piecewise_schedule(schedule_pieces)
+
+    def cosine_schedule(self, initial_lr=3e-4, final_lr=1e-4, total_timesteps=1e6): # check if this cosine annealing
+        return schedules.cosine_schedule(initial_lr, final_lr, total_timesteps)
+
+    def linear_warmup_schedule(self):
+        warmup_timesteps = 1e5
+        learning_rate_schedule = schedules.linear_schedule(9e5, 0, 3e-4)
+        learning_rate = schedules.linear_warmup(learning_rate_schedule, warmup_timesteps, 0)
+        return learning_rate
+
+    def combined_scheduler(self):
+        pass
+
+
+
+#####################################################################
+##                        POLICY(policy)                           ##
+#####################################################################
+# Categorical
+class PolicySelector:
+    def __init__(self):
+        pass
+
+    def opt(self, trial):
+        return "MlpPolicy"
+        
+    def default(self):
+        return self.MlpPolicy()
+
+    def MlpPolicy(self):
+        return "MlpPolicy"
+    
+    def CnnPolicy(self):
+        return "CnnPolicy"
+
+    def MultiInputPolicy(self):
+        return "MultiInputPolicy"
+    
+
+#####################################################################
+##                  UPDATE PER STEPS (n_steps)                     ##
+#####################################################################
+#Refine the search space --> dont just do 2^i in between also there might be peak
+
+class StepsPerUpdate:
+    
+    def __init__(self):
+        self.max_size = 2048
+
+        pass
+
+    def opt(self, trial):
+        return 2 ** trial.suggest_int("n_steps", 10, 12)
+
+    def default(self):
+        return random.randint(1, self.max_size)
+
+#####################################################################
+##                      BATCH SIZE (batch_size)                    ##
+#####################################################################
+
+class BatchSize:
+    def __init__(self):
+        self.max_size = 64
+        pass
+
+    def opt(self, trial):
+        return 2 ** trial.suggest_int("batch_size", 5, 8)
+    
+    def default(self):
+        self.max_size=26
+        return random.randint(1, self.max_size)
+
+#####################################################################
+##    No. OF EPOCHS WHEN OPTIMIZING SURROGATE LOSS (n_epochs)      ##
+#####################################################################
+# Higher the better
+
+class NoOfEpochs:
+    def __init__(self):
+        self.max_size = 10
+        pass
+
+    def opt(self, trial):
+        return trial.suggest_int("n_epochs", 5, 10)
+    
+    def default(self):
+        return random.randint(1, self.max_size)
+
+
+#####################################################################
+##                      DISCOUNT FACTOR (gamma)                    ##
+#####################################################################
+
+class DiscountFactor:
+    def __init__(self):
+        self.max_size = 10
+    
+    def opt(self, trial):
+        return trial.suggest_float("gamma", 0.9, 0.999)
+    
+    def default(self):
+        return random.random()
+
+#####################################################################
+##    BIAS-VARIANCE TRADEOFF for GENERALIZED ADVANTAGE ESTIMATOR   ##
+#####################################################################
+
+class BiasVarianceTradeoff:
+    def __init__(self):
+        pass
+
+    def opt(self, trial):
+        return trial.suggest_float("bvtradeoff", 0.9, 0.99)
+    
+    def default(self):
+        return random.random()
+
+#####################################################################
+##                  CLIPPING FUNCTION (clip_range)                 ##
+#####################################################################
+# Can be a function of progress
+
+class ClipRange:
+    def __init__(self):
+        pass
+
+    def opt(self, trial):
+        return trial.suggest_float("clip_range", 0.1, 0.3)
+    
+    def default(self):
+        return random.random()
+
+
+#####################################################################
+##       CLIPPING RANGE for Value Function (clip_range_vf)         ##
+#####################################################################
+
+class ClipRangeVF:
+    def __init__(self):
+        pass
+
+    def opt(self, trial):
+        return trial.suggest_float("clip_range_vf", 0.1, 0.3)
+    
+    def default(self):
+        return random.random()
+
+
+#####################################################################
+##             NORMALIZE ADVANTAGE(normalize_advantage)            ##
+#####################################################################
+
+class NormalizeAdvantage:
+    def __init__(self):
+        pass
+    
+    def opt(self, trial):
+        return False
+
+    def default(self):
+        return True
+
+
+#####################################################################
+##                  ENTROPY COEFFICIENT (ent_coef)                 ##
+#####################################################################
+
+class EntropyCoefficient:
+    def __init__(self):
+        pass
+
+    def opt(self, trial):
+        return trial.suggest_float("ent_coef", 0, 0.01)
+    
+    def default(self):
+        return 0
+
+
+#####################################################################
+##            VALUE FUNCTION COEFFICIENT (vf_coef)                 ##
+#####################################################################
+
+class ValueFunctionCoefficient:
+    def __init__(self):
+        pass
+
+    def opt(self, trial):
+        return trial.suggest_float("vf_coef", 0.25, 0.75)
+    
+    def default(self):
+        return 0.5
+
+#####################################################################
+##       MAXIMUM VALUE for Gradient Clipping (max_grad_norm)       ##
+#####################################################################
+
+class MaxGradNorm:
+    def __init__(self):
+        pass
+
+    def opt(self, trial):
+        return trial.suggest_float("max_grad_norm", 0.3, 1)
+    
+    def default(self):
+        return 0.5
+
+
+#####################################################################
+##        USE ? Generalized State Dependent Exploration            ##
+#####################################################################
+
+class BoolGeneralizedStateDependentExploration:
+    def __init__(self):
+        pass
+
+    def opt(self, trial):
+        return False
+    
+    def default(self):
+        return False
+
+
+#####################################################################
+##             SAMPLE NEW NOISE MATRIX(sde_sample_freq)            ##
+#####################################################################
+
+class SDESampleFrequency:
+    def __init__(self):
+        pass
+
+    def opt(self, trial):
+        return -1
+    
+    def default(self):
+        return -1
+
+
+#####################################################################
+##                LIMIT KL-Divergence (target_kl)                  ##
+#####################################################################
+
+class TargetKL:
+    def __init__(self):
+        pass
+
+    def opt(self, trial):
+        return trial.suggest_float("target_kl", 0.01, 0.05)
+    
+    def default(self):
+        return None
+
+
+#####################################################################
+##                      TENSORBOARD_LOG                            ##
+#####################################################################
+
+# class TensorBoardLog:
+#     def __init__(self):
+#         pass
+
+#     def opt(self, trial):
+#         return 
+    
+#     def default(self):
+#         return None
+
+
+#####################################################################
+##                   POLICY_KWARGS (policy_kwargs)                 ##
+#####################################################################
+
+class PolicyKWargs:
+    def __init__(self):
+        pass
+
+    def opt(self, trial):
+        return None
+    
+    def default(self):
+        return None
+
+
+#####################################################################
+##                           SEED (seed)                           ##
+#####################################################################
+
+class Seed:
+    def __init__(self):
+        pass
+
+    def opt(self, trial):
+        return trial.suggest_int("seed", 1, 10)
+    
+    def default(self):
+        return None
+
+
+all_policy = [
+    "MlpPolicy",
+    "CnnPolicy",
+    "MultiInputPolicy",
+]
+
+# ~!@ environment is fixed {maybe we can find stronger relations between hyperparameters across other environments}
+    # Very important idea
+
+
+all_classes_in_this_file = [
+    'BatchSize',
+    'BiasVarianceTradeoff',
+    'BoolGeneralizedStateDependentExploration',
+    'ClipRange',
+     'ClipRangeVF',
+      'DiscountFactor',
+       'EntropyCoefficient',
+        'Learning_Rate_Scheduler',
+         'MaxGradNorm',
+          'MegaHandler',
+           'NoOfEpochs',
+            'NormalizeAdvantage',
+             'PolicyKWargs',
+              'PolicySelector',
+               'SDESampleFrequency',
+                'Seed',
+                 'StepsPerUpdate',
+                  'TargetKL',
+                   'TensorBoardLog',
+                    'ValueFunctionCoefficient'
+]
+
+def request_next_HypConfig(env, tb_logs):
+    hyps = dict()
+
+    hyps["policy"] = PolicySelector().default()
+    hyps["env"] = env
+    hyps["learning_rate"] = Learning_Rate_Scheduler().default()
+    hyps["n_steps"] = StepsPerUpdate().default()
+    hyps["batch_size"] = BatchSize().default()
+    hyps["n_epochs"] = NoOfEpochs().default()
+    hyps["gamma"] = DiscountFactor().default()
+    hyps["gae_lambda"] = BiasVarianceTradeoff().default()
+    hyps["clip_range"] = ClipRange().default()
+    hyps["clip_range_vf"] = ClipRangeVF().default()
+    hyps["normalize_advantage"] = NormalizeAdvantage().default()
+    hyps["ent_coef"] = EntropyCoefficient().default()
+    hyps["vf_coef"] = ValueFunctionCoefficient().default()
+    hyps["max_grad_norm"] = MaxGradNorm().default()
+    hyps["use_sde"] = BoolGeneralizedStateDependentExploration().default()
+    hyps["sde_sample_freq"] = SDESampleFrequency().default()
+    hyps["target_kl"] = TargetKL().default()
+    hyps["tensorboard_log"] = tb_logs
+    hyps["policy_kwargs"] = PolicyKWargs().default()
+    hyps["verbose"] = 0
+    hyps["seed"] = Seed().default()
+    hyps["device"] = 'auto'
+    hyps["_init_setup_model"] = True
+
+    return hyps
+
+
+
+
+
+#####################################################################
+##            MEGA HANDLER (handler of all classes below)          ##
+#####################################################################
+class MegaHandler:
+    def __init__(slef):
+        pass
+
+    def request_next_HypConfig(self, trial, env, tb_logs):
+        hyps = dict()
+
+        hyps["policy"] = PolicySelector().opt(trial)
+        hyps["env"] = env
+        hyps["learning_rate"] = Learning_Rate_Scheduler().opt(trial)
+        hyps["n_steps"] = StepsPerUpdate().opt(trial)
+        hyps["batch_size"] = BatchSize().opt(trial)
+        hyps["n_epochs"] = NoOfEpochs().opt(trial)
+        hyps["gamma"] = DiscountFactor().opt(trial)
+        hyps["gae_lambda"] = BiasVarianceTradeoff().opt(trial)
+        hyps["clip_range"] = ClipRange().opt(trial)
+        hyps["clip_range_vf"] = ClipRangeVF().opt(trial)
+        hyps["normalize_advantage"] = NormalizeAdvantage().opt(trial)
+        hyps["ent_coef"] = EntropyCoefficient().opt(trial)
+        hyps["vf_coef"] = ValueFunctionCoefficient().opt(trial)
+        hyps["max_grad_norm"] = MaxGradNorm().opt(trial)
+        hyps["use_sde"] = BoolGeneralizedStateDependentExploration().opt(trial)
+        hyps["sde_sample_freq"] = SDESampleFrequency().opt(trial)
+        hyps["target_kl"] = TargetKL().opt(trial)
+        hyps["tensorboard_log"] = tb_logs
+        hyps["policy_kwargs"] = PolicyKWargs().opt(trial)
+        hyps["verbose"] = 0
+        hyps["seed"] = 1
+        hyps["device"] = 'auto'
+        hyps["_init_setup_model"] = True
+
+        return hyps

Trainer.py

@@ -0,0 +1,68 @@
+import os
+import datetime
+import gym
+import json
+
+from stable_baselines3 import PPO
+import optuna
+
+def check_json_exists(filename):
+    try:
+        open(filename, 'r')
+    except:
+        main = {
+            "env" : "RoadRunner-v0",
+            "algo" : "PPO",
+            "policy" : "MlpPolicy",
+            "counter" : 0,
+        }
+        with open(filename, "w") as f:
+            json.dump(main, f)
+
+check_json_exists("previous_request.json")
+
+class MegaTrainer:
+    def __init__(self, env_name, algo):
+        with open("previous_request.json", "r") as f:
+            self.data = json.loads(f.read())
+
+        self.env_name = env_name
+        self.algorithm = algo
+        self.counter = self.data["counter"] + 1
+
+        self.env = gym.make(self.env_name)
+        self.env.reset()
+
+        self.data["counter"] += 1
+        with open("previous_request.json", "w") as f:
+            json.dump(self.data, f)
+        
+    def make_directories(self, models_dir, logdir):
+        if(not os.path.exists(models_dir)):
+            os.makedirs(models_dir)
+
+        if(not os.path.exists(logdir)):
+            os.makedirs(logdir)
+
+    def learn(self, timestep, iterations, hyps, directory, trial):
+        model_dir = f"model/{directory}/{self.counter}"
+        logdir = f"logs/{directory}"
+        self.make_directories(model_dir, logdir)
+
+        model = PPO(**hyps)
+
+        for i in range(1, iterations+1):
+            if(trial.should_prune()):
+                raise optuna.TrialPruned()
+            dt = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S-%f")
+            model.learn(total_timesteps=timestep, reset_num_timesteps=False, tb_log_name=str(self.counter))
+            model.save(model_dir+f"/{dt}_{timestep*i}")
+
+        return self.counter
+
+
+
+
+
+if(__name__=="__main__"):
+    load_from_json("previous_request.json")

runner_trainer.py

@@ -0,0 +1,219 @@
+import plotly.offline as pyo
+
+import logging
+import sys
+
+import optuna
+from sklearn import model_selection
+
+from scipy.stats import randint, uniform
+
+import gym
+
+from Trainer import *
+from PPO_HypConfig import *
+from Loader import *
+
+all_optimizers  =[
+    'optuna',
+    'sklearn'
+]
+
+class Dummy:
+    def __init__(self):
+        pass
+
+    def fit(self):
+        pass
+
+class OptunaTuner:
+    def __init__(self, env_name, algo, directory, optimizer, ts, iterations, n_trials, counter):
+        self.env_name = env_name
+        self.algorithm = algo
+        self.directory = directory
+        self.optimizer = optimizer
+        self.timestep = ts
+        self.iterations = iterations
+        self.n_trials = n_trials
+        self.exe_number = counter
+
+    def create_study_db(self):
+        optuna.logging.get_logger("optuna").addHandler(logging.StreamHandler(sys.stdout))
+        study_name = f"study-{self.exe_number}"  # Unique identifier of the study.
+        
+        storage_dir = f"logs/{self.directory}"
+        if(not os.path.exists(storage_dir)):
+            os.makedirs(storage_dir)
+        storage_name = f"sqlite:///{storage_dir}/{study_name}.db"
+
+        return study_name, storage_name
+    
+    def select_pruner(self):
+        return None
+
+    def select_sampler(self):
+        return None
+    
+    def study_loader(self):
+        study_name = f"study-{self.exe_number}"  # Unique identifier of the study.
+        storage_dir = f"logs/{self.directory}"
+        storage_name = f"sqlite:///{storage_dir}/{study_name}.db"
+
+        study = optuna.load_study(
+            study_name=study_name,
+            storage=storage_name
+        )
+
+        trials_df = study.trials_dataframe()
+        print(trials_df.head())
+
+        return study
+    
+    def study_summaries(self):
+        study_name = f"study-{self.exe_number}"  # Unique identifier of the study.
+        storage_dir = f"logs/{self.directory}"
+        storage_name = f"sqlite:///{storage_dir}/{study_name}.db"
+
+        summary = optuna.get_all_study_summaries(
+            storage_name, include_best_trial=True
+        )
+        print(summary)
+        for i,j in enumerate(summary):
+            print(f"-----------STUDY-{i}-----------")
+            for attribute in [attr for attr in dir(j) if not attr.startswith('__')]:
+                if(attribute=="system_attrs" or attribute=="user_attrs"):
+                    continue
+                print(attribute, ':', getattr(j, attribute))
+            print()
+        pass
+
+    def under_testing_py(self):
+        study = self.study_loader()
+        trial = study.ask()
+        try:
+            optuna.testing.objectives.fail_objective(trial)
+        except ValueError:
+            print("Objective function failed!")
+
+        ## To Prune
+        optuna.testing.objectives.pruned_objective(trial)
+
+        ## Dummy trial
+        study = optuna.testing.visualization.prepare_study_with_trials()
+
+    def under_terminators(self):
+
+        pass
+    
+    def return_study(self, bool_store_study=True, bool_select_pruner=True, bool_select_sampler=True):
+        study_name, storage_name, selected_pruner, selected_sampler = [None] * 4
+        if(bool_store_study):
+            study_name, storage_name = self.create_study_db()
+
+        if(bool_select_pruner):
+            selected_pruner = self.select_pruner()
+        
+        if(bool_select_sampler):
+            selected_sampler = self.select_sampler()
+        
+        lie = storage_name is not None
+
+        study = optuna.create_study(
+                storage=storage_name,
+                sampler=selected_sampler,
+                pruner=selected_pruner,
+                study_name=study_name,
+                direction='maximize',
+                load_if_exists=lie,
+                directions=None
+            )
+        return study
+
+    def call_optuna(self, store_study=True):
+        study = self.return_study()
+        study.optimize(self.objective, n_trials=self.n_trials)
+        
+    def objective(self, trial):
+        m = MegaTrainer(self.env_name, self.algorithm)
+        counter = m.learn(
+            self.timestep,
+            self.iterations,
+            MegaHandler().request_next_HypConfig(trial, m.env, f"logs/{self.directory}"),
+            self.directory,
+            trial=trial
+        )
+
+        m_load  = MegaLoader(self.env_name, self.algorithm, self.directory)
+        return m_load.load(counter, self.n_trials)["total_return"]
+
+    def visul(self):
+        study = self.study_loader()
+        if(not "_optimization_history.py"):
+            pyo.plot(optuna.visualization.plot_optimization_history(study))
+        
+        if(not "_timeline.py"):
+            pyo.plot(optuna.visualization.plot_timeline(study))
+        
+        if(not "_slice.py"):
+            pyo.plot(optuna.visualization.plot_slice(study))
+        
+        if(not "_pareto_front.py"): ## this is used for multi-objective study
+            pyo.plot(optuna.visualization.plot_pareto_front(study))
+        
+        if(not "_param_importances.py"):
+            pyo.plot(optuna.visualization.plot_param_importances(study))
+        
+        if(not "_parallel_coordinate.py"):
+            pyo.plot(optuna.visualization.plot_parallel_coordinate(study))
+            
+        if(not "_intermediate_values.py"):
+            pyo.plot(optuna.visualization.plot_intermediate_values(study))
+        
+        if(not "_edf.py"):
+            pyo.plot(optuna.visualization.plot_edf(study))
+            
+        if(not "_contour.py"):
+            pyo.plot(optuna.visualization.plot_contour(study))
+
+class HyperPilotRL:
+    def __init__(self, env_name, algo, optimizer, timesteps, iterations, n_trials, counter):
+        self.env_name = env_name
+        self.algorithm = algo
+        self.directory = f"{optimizer}/{counter}/{env_name}/{algo}"
+        self.optimizer = optimizer
+        self.timestep = timesteps
+        self.iterations = iterations
+        self.n_trials = n_trials
+        self.execution_number = counter
+
+        if(self.optimizer == "optuna"):
+            o = OptunaTuner(env_name, algo, self.directory, optimizer, timesteps, iterations, n_trials, counter)
+            o.call_optuna()
+            o.study_summaries()
+            # o.visul()
+        elif(self.optimizer == "default"):
+            pass
+
+        else:
+            raise NameError("Enter optimizer is not supported.\nAvailable optimizers are", all_optimizers)
+
+        ## optimizer = ['optuna']
+
+    def learning_curve():
+        pass
+
+    def cutoff_learn_according_to_max_time_allocation():
+        pass
+
+if(__name__=="__main__"):
+    h = HyperPilotRL(
+        "CartPole-v1",
+        "PPO",
+        "optuna",
+        timesteps=1000,
+        iterations=100,
+        n_trials=10,
+        counter=102644
+    )
+
+