Instruction:
Write a Python 3 function that can
create a function that converts the binary solution to parameters.

Response:
def _make_hyperparameter_decode_func(locked_values, meta_parameters):
    """Create a function that converts the binary solution to parameters."""

    # Locked parameters are also returned by decode function, but are not
    # based on solution

    def decode(solution):
        """Convert solution into dict of hyperparameters."""
        # Start with out stationary (locked) paramaters
        hyperparameters = copy.deepcopy(locked_values)

        # Obtain moving hyperparameters from binary solution
        index = 0
        for name, parameters in meta_parameters.iteritems():
            # Obtain binary for this hyperparameter
            binary_size = parameters['binary_size']
            binary = solution[index:index + binary_size]
            index += binary_size  # Just index to start of next hyperparameter

            # Decode binary
            if parameters['type'] == 'discrete':
                i = helpers.binary_to_int(
                    binary, upper_bound=len(parameters['values']) - 1)
                value = parameters['values'][i]
            elif parameters['type'] == 'int':
                value = helpers.binary_to_int(
                    binary,
                    lower_bound=parameters['min'],
                    upper_bound=parameters['max'])
            elif parameters['type'] == 'float':
                value = helpers.binary_to_float(
                    binary,
                    lower_bound=parameters['min'],
                    upper_bound=parameters['max'])
            else:
                raise ValueError(
                    'Parameter type "{}" does not match known values'.format(
                        parameters['type']))

            # Store value
            hyperparameters[name] = value

        return hyperparameters

    return decode


Instruction:
Can you tell what is the following Python 3 function doing
def _meta_fitness_func(parameters,
                       _optimizer,
                       _problems,
                       _master_fitness_dict,
                       _runs=20):
    """Test a metaheuristic with parameters encoded in solution.

    Our goal is to minimize number of evaluation runs until a solution is found,
    while maximizing chance of finding solution to the underlying problem
    NOTE: while meta optimization requires a 'known' solution, this solution
    can be an estimate to provide the meta optimizer with a sense of progress.
    """
    # Create the optimizer with parameters encoded in solution
    optimizer = copy.deepcopy(_optimizer)
    optimizer._set_hyperparameters(parameters)
    optimizer.logging = False

    # Preload fitness dictionary from master, and disable clearing dict
    # NOTE: master_fitness_dict will be modified inline, and therefore,
    # we do not need to take additional steps to update it
    if _master_fitness_dict != None:  # None means low memory mode
        optimizer.clear_cache = False
        optimizer._Optimizer__encoded_cache = _master_fitness_dict

    # Because metaheuristics are stochastic, we run the optimizer multiple times,
    # to obtain an average of performance
    all_evaluation_runs = []
    solutions_found = []
    for _ in range(_runs):
        for problem in _problems:
            # Get performance for problem
            optimizer.optimize(problem)
            all_evaluation_runs.append(optimizer.fitness_runs)
            if optimizer.solution_found:
                solutions_found.append(1.0)
            else:
                solutions_found.append(0.0)

    # Our main goal is to minimize time the optimizer takes
    fitness = 1.0 / helpers.avg(all_evaluation_runs)

    # Optimizer is heavily penalized for missing solutions
    # To avoid 0 fitness
    fitness = fitness * helpers.avg(solutions_found)**2 + 1e-19

    return fitness

Response: Test a metaheuristic with parameters encoded in solution.


Instruction:
Can you tell what is the following Python 3 function doing
def copy(self,
             fitness_function=None,
             decode_function=None,
             fitness_args=None,
             decode_args=None,
             fitness_kwargs=None,
             decode_kwargs=None):
        """Return a copy of this problem.

        Optionally replace this problems arguments with those passed in.
        """
        if fitness_function is None:
            fitness_function = self._fitness_function
        if decode_function is None:
            decode_function = self._decode_function
        if fitness_args is None:
            fitness_args = self._fitness_args
        if decode_args is None:
            decode_args = self._decode_args
        if fitness_kwargs is None:
            fitness_kwargs = self._fitness_kwargs
        if decode_kwargs is None:
            decode_kwargs = self._decode_kwargs

        return Problem(
            fitness_function,
            decode_function=decode_function,
            fitness_args=fitness_args,
            decode_args=decode_args,
            fitness_kwargs=fitness_kwargs,
            decode_kwargs=decode_kwargs)

Response: Returns a copy of this problem.


Instruction:
Given the following Python 3 function, write the documentation
def get_fitness(self, solution):
        return self._fitness_function(solution, *self._fitness_args,
                                      **self._fitness_kwargs)

Response: Return fitness for the given solution.


Instruction:
Explain what the following Python 3 code does
def decode_solution(self, encoded_solution):
        return self._decode_function(encoded_solution, *self._decode_args,
                                     **self._decode_kwargs)

Response: Return solution from an encoded representation.


Instruction:
Here you have a function in Python 3, explain what it does
def optimize(self, problem, max_iterations=100, max_seconds=float('inf'),
                 cache_encoded=True, cache_solution=False, clear_cache=True,
                 logging_func=_print_fitnesses,
                 n_processes=0):
        """Find the optimal inputs for a given fitness function.

        Args:
            problem: An instance of Problem. The problem to solve.
            max_iterations: The number of iterations to optimize before stopping.
            max_seconds: Maximum number of seconds to optimize for, before stopping.
                Note that condition is only checked one per iteration,
                meaning optimization can take more than max_seconds,
                especially if fitnesses take a long time to calculate.
            cache_encoded: bool; Whether or not to cache fitness of encoded strings.
                Encoded strings are produced directly by the optimizer.
                If an encoded string is found in cache, it will not be decoded.
            cache_solution: bool; Whether or not to cache fitness of decoded solutions.
                Decoded solution is provided by problems decode function.
                If problem does not provide a hash solution function,
                Various naive hashing methods will be attempted, including:
                    tuple, tuple(sorted(dict.items)), str.
            clear_cache: bool; Whether or not to reset cache after optimization.
                Disable if you want to run optimize multiple times on the same problem.
            logging_func: func/None; Function taking:
                iteration, population, solutions, fitnesses, best_solution, best_fitness
                Called after every iteration.
                Use for custom logging, or set to None to disable logging.
                Note that best_solution and best_fitness are the best of all iterations so far.
            n_processes: int; Number of processes to use for multiprocessing.
                If <= 0, do not use multiprocessing.

        Returns:
            object; The best solution, after decoding.
        """
        if not isinstance(problem, Problem):
            raise TypeError('problem must be an instance of Problem class')

        # Prepare pool for multiprocessing
        if n_processes > 0:
            try:
                pool = multiprocessing.Pool(processes=n_processes)
            except NameError:
                raise ImportError(
                    'pickle, dill, or multiprocessing library is not available.'
                )
        else:
            pool = None

        # Set first, incase optimizer uses _max_iterations in initialization
        self.__max_iterations = max_iterations

        # Initialize algorithm
        self._reset()

        best_solution = {'solution': None, 'fitness': None}
        population = self.initial_population()
        try:
            # Begin optimization loop
            start = time.clock()
            for self.iteration in itertools.count(1):  # Infinite sequence of iterations
                # Evaluate potential solutions
                solutions, fitnesses, finished = self._get_fitnesses(
                    problem,
                    population,
                    cache_encoded=cache_encoded,
                    cache_solution=cache_solution,
                    pool=pool)

                # If the best fitness from this iteration is better than
                # the global best
                best_index, best_fitness = max(
                    enumerate(fitnesses), key=operator.itemgetter(1))
                if best_fitness > best_solution['fitness']:
                    # Store the new best solution
                    best_solution['fitness'] = best_fitness
                    best_solution['solution'] = solutions[best_index]

                if logging_func:
                    logging_func(self.iteration, population, solutions,
                                 fitnesses, best_solution['solution'],
                                 best_solution['fitness'])

                # Break if solution found
                if finished:
                    self.solution_found = True
                    break

                # Break if out of time
                if time.clock() - start >= max_seconds:
                    break

                # Break if out of iterations
                if self.iteration >= max_iterations:
                    break

                # Continue optimizing
                population = self.next_population(population, fitnesses)

            # Store best internally, before returning
            self.best_solution = best_solution['solution']
            self.best_fitness = best_solution['fitness']

        finally:
            # Clear caches
            if clear_cache:
                # Clear caches from memory
                self.__encoded_cache = {}
                self.__solution_cache = {}

                # Reset encoded, and decoded key functions
                self._get_encoded_key = self._get_encoded_key_type
                self._get_solution_key = self._get_solution_key_type

            # Clean up multiprocesses
            try:
                pool.terminate()  # Kill outstanding work
                pool.close()  # Close child processes
            except AttributeError:
                # No pool
                assert pool is None

        return self.best_solution

Response: This function will find the optimal inputs for a given fitness function.


Instruction:
Can you tell what is the following Python 3 function doing
def _reset_bookkeeping(self):
        self.iteration = 0
        self.fitness_runs = 0
        self.best_solution = None
        self.best_fitness = None
        self.solution_found = False

Response: Reset the bookkeeping parameters to initial values.


Instruction:
Write a Python 3 script to
get the fitness for every solution in a population.

Response:
def _get_fitnesses(self,
                       problem,
                       population,
                       cache_encoded=True,
                       cache_solution=False,
                       pool=None):
        """Get the fitness for every solution in a population.

        Args:
            problem: Problem; The problem that defines fitness.
            population: list; List of potential solutions.
            pool: None/multiprocessing.Pool; Pool of processes for parallel
                decoding and evaluation.
        """
        fitnesses = [None] * len(population)

        #############################
        # Decoding
        #############################
        if cache_encoded:
            try:
                encoded_keys = map(self._get_encoded_key, population)

                # Get all fitnesses from encoded_solution cache
                to_decode_indices = []
                for i, encoded_key in enumerate(encoded_keys):
                    try:
                        fitnesses[i] = self.__encoded_cache[encoded_key]
                        # Note that this fitness will never be better than the current best
                        # because we have already evaluted it,
                        # Therefore, we do not need to worry about decoding the solution
                    except KeyError:  # Cache miss
                        to_decode_indices.append(i)

            except UnhashableError:  # Cannot hash encoded solution
                encoded_keys = None
                to_decode_indices = range(len(population))
        else:
            encoded_keys = None
            to_decode_indices = range(len(population))

        # Decode all that need to be decoded, and combine back into list the same length
        # as population
        if encoded_keys is None:
            to_decode_keys = None
        else:
            to_decode_keys = [encoded_keys[i] for i in to_decode_indices]

        solutions = [None] * len(population)
        for i, solution in zip(to_decode_indices,
                               self._pmap(
                                   problem.decode_solution,
                                   [population[i] for i in to_decode_indices],
                                   to_decode_keys,
                                   pool)):
            solutions[i] = solution

        #############################
        # Evaluating
        #############################
        if cache_solution:
            try:
                # Try to make solutions hashable
                # Use user provided hash function if available
                if problem.hash_solution:
                    hash_solution_func = problem.hash_solution
                else:
                    # Otherwise, default to built in "smart" hash function
                    hash_solution_func = self._get_solution_key

                solution_keys = [
                    hash_solution_func(solution)
                    # None corresponds to encoded_solutions found in cache
                    if solution is not None else None for solution in solutions
                ]

                # Get all fitnesses from solution cache
                to_eval_indices = []
                for i, solution_key in enumerate(solution_keys):
                    if solution_key is not None:  # Otherwise, fitness already found in encoded cache
                        try:
                            fitnesses[i] = self.__solution_cache[solution_key]
                        except KeyError:  # Cache miss
                            to_eval_indices.append(i)

            except UnhashableError:  # Cannot hash solution
                solution_keys = None
                to_eval_indices = to_decode_indices[:]
        else:
            solution_keys = None
            to_eval_indices = to_decode_indices[:]

        # Evaluate all that need to be evaluated, and combine back into fitnesses list
        if solution_keys is None:
            if encoded_keys is None:
                # No way to detect duplicates
                to_eval_keys = None
            else:
                # Cannot use decoded keys, default to encoded keys
                to_eval_keys = [encoded_keys[i] for i in to_eval_indices]
        else:
            to_eval_keys = [solution_keys[i] for i in to_eval_indices]

        finished = False
        eval_bookkeeping = {}
        for i, fitness_finished in zip(to_eval_indices,
                                       self._pmap(
                                           problem.get_fitness,
                                           [solutions[i] for i in to_eval_indices],
                                           to_eval_keys,
                                           pool,
                                           bookkeeping_dict=eval_bookkeeping)):
            # Unpack fitness_finished tuple
            try:
                fitness, maybe_finished = fitness_finished
                if maybe_finished:
                    finished = True
            except TypeError:  # Not (fitness, finished) tuple
                fitness = fitness_finished

            fitnesses[i] = fitness

        #############################
        # Finishing
        #############################
        # Bookkeeping
        # keep track of how many times fitness is evaluated
        self.fitness_runs += len(eval_bookkeeping['key_indices'])  # Evaled once for each unique key

        # Add evaluated fitnesses to caches (both of them)
        if cache_encoded and encoded_keys is not None:
            for i in to_decode_indices:  # Encoded cache misses
                self.__encoded_cache[encoded_keys[i]] = fitnesses[i]
        if cache_solution and solution_keys is not None:
            for i in to_eval_indices:  # Decoded cache misses
                self.__solution_cache[solution_keys[i]] = fitnesses[i]

        # Return
        # assert None not in fitnesses  # Un-comment for debugging
        return solutions, fitnesses, finished


Instruction:
Can you tell what is the following Python 3 function doing
def _pmap(self, func, items, keys, pool, bookkeeping_dict=None):
        if keys is not None:  # Otherwise, cannot hash items
            # Remove duplicates first (use keys)
            # Create mapping (dict) of key to list of indices
            key_indices = _duplicates(keys).values()
        else:  # Cannot hash items
            # Assume no duplicates
            key_indices = [[i] for i in range(len(items))]

        # Use only the first of duplicate indices in decoding
        if pool is not None:
            # Parallel map
            results = pool.map(
                functools.partial(_unpickle_run, pickle.dumps(func)),
                [items[i[0]] for i in key_indices])
        else:
            results = map(func, [items[i[0]] for i in key_indices])

        # Add bookkeeping
        if bookkeeping_dict is not None:
            bookkeeping_dict['key_indices'] = key_indices

        # Combine duplicates back into list
        all_results = [None] * len(items)
        for indices, result in zip(key_indices, results):
            for j, i in enumerate(indices):
                # Avoid duplicate result objects in list,
                # in case they are used in functions with side effects
                if j > 0:
                    result = copy.deepcopy(result)
                all_results[i] = result

        return all_results

Response: Efficiently map func over all items.


Instruction:
Explain what the following Python 3 code does
def _set_hyperparameters(self, parameters):
        for name, value in parameters.iteritems():
            try:
                getattr(self, name)
            except AttributeError:
                raise ValueError(
                    'Each parameter in parameters must be an attribute. '
                    '{} is not.'.format(name))
            setattr(self, name, value)

Response: Set internal optimization parameters.


Instruction:
Can you generate the documentation for the following Python 3 function
def _get_hyperparameters(self):
        hyperparameters = {}
        for key in self._hyperparameters:
            hyperparameters[key] = getattr(self, key)
        return hyperparameters

Response: Get internal optimization parameters.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def optimize_hyperparameters(self,
                                 problems,
                                 parameter_locks=None,
                                 smoothing=20,
                                 max_iterations=100,
                                 _meta_optimizer=None,
                                 _low_memory=True):
        """Optimize hyperparameters for a given problem.

        Args:
            parameter_locks: a list of strings, each corresponding to a hyperparamter
                             that should not be optimized.
            problems: Either a single problem, or a list of problem instances,
                     allowing optimization based on multiple similar problems.
            smoothing: int; number of runs to average over for each set of hyperparameters.
            max_iterations: The number of iterations to optimize before stopping.
            _low_memory: disable performance enhancements to save memory
                         (they use a lot of memory otherwise).
        """
        if smoothing <= 0:
            raise ValueError('smoothing must be > 0')

        # problems supports either one or many problem instances
        if isinstance(problems, collections.Iterable):
            for problem in problems:
                if not isinstance(problem, Problem):
                    raise TypeError(
                        'problem must be Problem instance or list of Problem instances'
                    )
        elif isinstance(problems, Problem):
            problems = [problems]
        else:
            raise TypeError(
                'problem must be Problem instance or list of Problem instances'
            )

        # Copy to avoid permanent modification
        meta_parameters = copy.deepcopy(self._hyperparameters)

        # First, handle parameter locks, since it will modify our
        # meta_parameters dict
        locked_values = _parse_parameter_locks(self, meta_parameters,
                                               parameter_locks)

        # We need to know the size of our chromosome,
        # based on the hyperparameters to optimize
        solution_size = _get_hyperparameter_solution_size(meta_parameters)

        # We also need to create a decode function to transform the binary solution
        # into parameters for the metaheuristic
        decode = _make_hyperparameter_decode_func(locked_values,
                                                  meta_parameters)

        # A master fitness dictionary can be stored for use between calls
        # to meta_fitness
        if _low_memory:
            master_fitness_dict = None
        else:
            master_fitness_dict = {}

        additional_parameters = {
            '_optimizer': self,
            '_problems': problems,
            '_runs': smoothing,
            '_master_fitness_dict': master_fitness_dict,
        }
        META_FITNESS = Problem(
            _meta_fitness_func,
            decode_function=decode,
            fitness_kwargs=additional_parameters)
        if _meta_optimizer is None:
            # Initialize default meta optimizer
            # GenAlg is used because it supports both discrete and continous
            # attributes
            from optimal import GenAlg

            # Create metaheuristic with computed decode function and soltuion
            # size
            _meta_optimizer = GenAlg(solution_size)
        else:
            # Adjust supplied metaheuristic for this problem
            _meta_optimizer._solution_size = solution_size

        # Determine the best hyperparameters with a metaheuristic
        best_parameters = _meta_optimizer.optimize(
            META_FITNESS, max_iterations=max_iterations)

        # Set the hyperparameters inline
        self._set_hyperparameters(best_parameters)

        # And return
        return best_parameters

Response: Optimize hyperparameters for a given problem.


Instruction:
Implement a function in Python 3 to
compare a set of optimizers and problems.

Response:
def compare(optimizers, problems, runs=20, all_kwargs={}):
    """Compare a set of optimizers.

    Args:
        optimizers: list/Optimizer; Either a list of optimizers to compare,
            or a single optimizer to test on each problem.
        problems: list/Problem; Either a problem instance or a list of problem instances,
            one for each optimizer.
        all_kwargs: dict/list<dict>; Either the Optimizer.optimize keyword arguments
            for all optimizers, or a list of keyword arguments, one for each optimizer.
        runs: int; How many times to run each optimizer (smoothness)

    Returns:
        dict; mapping optimizer identifier to stats.
    """
    if not (isinstance(optimizers, collections.Iterable)
            or isinstance(problems, collections.Iterable)):
        raise TypeError('optimizers or problems must be iterable')

    # If optimizers is not a list, repeat into list for each problem
    if not isinstance(optimizers, collections.Iterable):
        optimizers = [copy.deepcopy(optimizers) for _ in range(len(problems))]

    # If problems is not a list, repeat into list for each optimizer
    if not isinstance(problems, collections.Iterable):
        problems = [copy.deepcopy(problems) for _ in range(len(optimizers))]

    # If all_kwargs is not a list, repeat it into a list
    if isinstance(all_kwargs, dict):
        all_kwargs = [all_kwargs] * len(optimizers)
    elif not isinstance(all_kwargs, collections.Iterable):
        raise TypeError('all_kwargs must be dict or list of dict')

    stats = {}
    key_counts = {}
    for optimizer, problem, kwargs in zip(optimizers, problems, all_kwargs):
        # For nice human readable dictionaries, extract useful names from
        # optimizer
        class_name = optimizer.__class__.__name__
        fitness_func_name = problem._fitness_function.__name__
        key_name = '{} {}'.format(class_name, fitness_func_name)

        # Keep track of how many optimizers of each class / fitness func
        # for better keys in stats dict
        try:
            key_counts[key_name] += 1
        except KeyError:
            key_counts[key_name] = 1

        # Foo 1, Foo 2, Bar 1, etc.
        key = '{} {}'.format(key_name, key_counts[key_name])

        print key + ': ',

        # Finally, get the actual stats
        stats[key] = benchmark(optimizer, problem, runs=runs, **kwargs)

        print

    return stats


Instruction:
Can you tell what is the following Python 3 function doing
def benchmark(optimizer, problem, runs=20, **kwargs):
    stats = {'runs': []}

    # Disable logging, to avoid spamming the user
    # TODO: Maybe we shouldn't disable by default?
    kwargs = copy.copy(kwargs)
    kwargs['logging_func'] = None

    # Determine effectiveness of metaheuristic over many runs
    # The stochastic nature of metaheuristics make this necessary
    # for an accurate evaluation
    for _ in range(runs):
        optimizer.optimize(problem, **kwargs)

        # Convert bool to number for mean and standard deviation calculations
        if optimizer.solution_found:
            finished_num = 1.0
        else:
            finished_num = 0.0

        stats_ = {
            'fitness': optimizer.best_fitness,
            'fitness_runs': optimizer.fitness_runs,
            'solution_found': finished_num
        }
        stats['runs'].append(stats_)

        # Little progress 'bar'
        print '.',

    # Mean gives a good overall idea of the metaheuristics effectiveness
    # Standard deviation (SD) shows consistency of performance
    _add_mean_sd_to_stats(stats)

    return stats

Response: Run an optimizer through a problem multiple times and return a dictionary of various statistics.


Instruction:
Write a Python 3 function that can
combine stats for multiple optimizers to obtain one mean and sd.

Response:
def aggregate(all_stats):
    """Combine stats for multiple optimizers to obtain one mean and sd.

    Useful for combining stats for the same optimizer class and multiple problems.

    Args:
        all_stats: dict; output from compare.
    """
    aggregate_stats = {'means': [], 'standard_deviations': []}
    for optimizer_key in all_stats:
        # runs is the mean, for add_mean_sd function
        mean_stats = copy.deepcopy(all_stats[optimizer_key]['mean'])
        mean_stats['name'] = optimizer_key
        aggregate_stats['means'].append(mean_stats)

        # also keep track of standard deviations
        sd_stats = copy.deepcopy(
            all_stats[optimizer_key]['standard_deviation'])
        sd_stats['name'] = optimizer_key
        aggregate_stats['standard_deviations'].append(sd_stats)

    _add_mean_sd_to_stats(aggregate_stats, 'means')

    return aggregate_stats


Instruction:
How would you implement a function in Python 3 that
obtains the mean of stats.

Response:
def _mean_of_runs(stats, key='runs'):
    """Obtain the mean of stats.

    Args:
        stats: dict; A set of stats, structured as above.
        key: str; Optional key to determine where list of runs is found in stats
    """

    num_runs = len(stats[key])
    first = stats[key][0]

    mean = {}
    for stat_key in first:
        # Skip non numberic attributes
        if isinstance(first[stat_key], numbers.Number):
            mean[stat_key] = sum(run[stat_key]
                                 for run in stats[key]) / float(num_runs)

    return mean


Instruction:
Can you generate the documentation for the following Python 3 function
def _sd_of_runs(stats, mean, key='runs'):

    num_runs = len(stats[key])
    first = stats[key][0]

    standard_deviation = {}
    for stat_key in first:
        # Skip non numberic attributes
        if isinstance(first[stat_key], numbers.Number):
            standard_deviation[stat_key] = math.sqrt(
                sum((run[stat_key] - mean[stat_key])**2
                    for run in stats[key]) / float(num_runs))

    return standard_deviation

Response: Obtain the standard deviation of stats.


Instruction:
How would you implement a function in Python 3 that
returns random binary string with given probabilities.

Response:
def _sample(probability_vec):
    """Return random binary string, with given probabilities."""
    return map(int,
               numpy.random.random(probability_vec.size) <= probability_vec)


Instruction:
Create a Python 3 function for
adjusting probabilities towards the best solution.

Response:
def _adjust_probability_vec_best(population, fitnesses, probability_vec,
                                 adjust_rate):
    """Shift probabilities towards the best solution."""
    best_solution = max(zip(fitnesses, population))[1]

    # Shift probabilities towards best solution
    return _adjust(probability_vec, best_solution, adjust_rate)


Instruction:
Here you have a function in Python 3, explain what it does
def _mutate_probability_vec(probability_vec, mutation_chance, mutation_adjust_rate):
    bits_to_mutate = numpy.random.random(probability_vec.size) <= mutation_chance
    probability_vec[bits_to_mutate] = _adjust(
        probability_vec[bits_to_mutate],
        numpy.random.random(numpy.sum(bits_to_mutate)), mutation_adjust_rate)

Response: Randomly adjusts the probability_vec.


Instruction:
Write a Python 3 function for
making a new population after each optimization iteration.

Response:
def next_population(self, population, fitnesses):
        """Make a new population after each optimization iteration.

        Args:
            population: The population current population of solutions.
            fitnesses: The fitness associated with each solution in the population
        Returns:
            list; a list of solutions.
        """
        # Update probability vector
        self._probability_vec = _adjust_probability_vec_best(
            population, fitnesses, self._probability_vec, self._adjust_rate)

        # Mutate probability vector
        _mutate_probability_vec(self._probability_vec, self._mutation_chance,
                                self._mutation_adjust_rate)

        # Return new samples
        return [
            _sample(self._probability_vec)
            for _ in range(self._population_size)
        ]


Instruction:
Make a summary of the following Python 3 code
def benchmark_multi(optimizer):
    # Get our benchmark stats
    all_stats = benchmark.compare(optimizer, PROBLEMS, runs=100)
    return benchmark.aggregate(all_stats)

Response: Benchmark an optimizer configuration on multiple functions.


Instruction:
Can you write a function in Python 3 where it
returns a random population drawn with regard to a set of probabilities

Response:
def _sample(probabilities, population_size):
    """Return a random population, drawn with regard to a set of probabilities"""
    population = []
    for _ in range(population_size):
        solution = []
        for probability in probabilities:
            # probability of 1.0: always 1
            # probability of 0.0: always 0
            if random.uniform(0.0, 1.0) < probability:
                solution.append(1)
            else:
                solution.append(0)
        population.append(solution)
    return population


Instruction:
Can you tell what is the following Python 3 function doing
def _chance(solution, pdf):
    # 1.0 - abs(bit - p) gives probability of bit given p
    return _prod([1.0 - abs(bit - p) for bit, p in zip(solution, pdf)])

Response: Return the chance of obtaining a solution from a pdf.


Instruction:
Implement a Python 3 function for
giving the value of a pdf.

Response:
def _pdf_value(pdf, population, fitnesses, fitness_threshold):
    """Give the value of a pdf.

    This represents the likelihood of a pdf generating solutions
    that exceed the threshold.
    """
    # Add the chance of obtaining a solution from the pdf
    # when the fitness for that solution exceeds a threshold
    value = 0.0
    for solution, fitness in zip(population, fitnesses):
        if fitness >= fitness_threshold:
            # 1.0 + chance to avoid issues with chance of 0
            value += math.log(1.0 + _chance(solution, pdf))

    # The official equation states that value is now divided by len(fitnesses)
    # however, this is unnecessary when we are only obtaining the best pdf,
    # because every solution is of the same size
    return value


Instruction:
Make a summary of the following Python 3 code
def _update_pdf(population, fitnesses, pdfs, quantile):
    # First we determine a fitness threshold based on a quantile of fitnesses
    fitness_threshold = _get_quantile_cutoff(fitnesses, quantile)

    # Then check all of our possible pdfs with a stochastic program
    return _best_pdf(pdfs, population, fitnesses, fitness_threshold)

Response: Find a better pdf based on fitnesses.


Instruction:
Write a Python 3 script to
return a floating point number between lower and upper bounds, from binary. Args: binary_list: list<int>; List of 0s and 1s. The number of bits in this list determine the number of possible values between lower and upper bound. Increase the size of binary_list for more precise floating points. lower_bound: Minimum value for output, inclusive. A binary list of 0s will have this value. upper_bound: Maximum value for output, inclusive. A binary list of 1s will have this value. Returns: float; A floating point number.

Response:
def binary_to_float(binary_list, lower_bound, upper_bound):
    """Return a floating point number between lower and upper bounds, from binary.

    Args:
        binary_list: list<int>; List of 0s and 1s.
            The number of bits in this list determine the number of possible
            values between lower and upper bound.
            Increase the size of binary_list for more precise floating points.
        lower_bound: Minimum value for output, inclusive.
            A binary list of 0s will have this value.
        upper_bound: Maximum value for output, inclusive.
            A binary list of 1s will have this value.

    Returns:
        float; A floating point number.
    """
    # Edge case for empty binary_list
    if binary_list == []:
        # With 0 bits, only one value can be represented,
        # and we default to lower_bound
        return lower_bound

    # A little bit of math gets us a floating point
    # number between upper and lower bound
    # We look at the relative position of
    # the integer corresponding to our binary list
    # between the upper and lower bound,
    # and offset that by lower bound
    return ((
        # Range between lower and upper bound
        float(upper_bound - lower_bound)
        # Divided by the maximum possible integer
        / (2**len(binary_list) - 1)
        # Times the integer represented by the given binary
        * binary_to_int(binary_list))
            # Plus the lower bound
            + lower_bound)


Instruction:
Given the following Python 3 function, write the documentation
def binary_to_int(binary_list, lower_bound=0, upper_bound=None):
    # Edge case for empty binary_list
    if binary_list == []:
        # With 0 bits, only one value can be represented,
        # and we default to lower_bound
        return lower_bound
    else:
        # The builtin int construction can take a base argument,
        # but it requires a string,
        # so we convert our binary list to a string
        integer = int(''.join([str(bit) for bit in binary_list]), 2)

    # Trim if over upper_bound
    if (upper_bound is not None) and integer + lower_bound > upper_bound:
        # Bounce back. Ex. w/ upper_bound = 2: [0, 1, 2, 2, 1, 0]
        return upper_bound - (integer % (upper_bound - lower_bound + 1))
    else:
        # Not over upper_bound
        return integer + lower_bound

Response: Return the base 10 integer corresponding to a binary list.


Instruction:
Implement a function in Python 3 to
return bit list representation of integer.

Response:
def _int_to_binary(integer, size=None):
    """Return bit list representation of integer.

    If size is given, binary string is padded with 0s, or clipped to the size.
    """
    binary_list = map(int, format(integer, 'b'))

    if size is None:
        return binary_list
    else:
        if len(binary_list) > size:
            # Too long, take only last n
            return binary_list[len(binary_list)-size:]
        elif size > len(binary_list):
            # Too short, pad
            return [0]*(size-len(binary_list)) + binary_list
        else:
            # Just right
            return binary_list


Instruction:
Make a summary of the following Python 3 code
def next_population(self, population, fitnesses):
        return common.make_population(self._population_size,
                                      self._generate_solution)

Response: Make a new population after each optimization iteration.


Instruction:
Explain what the following Python 3 code does
def _generate_solution(self):
        return common.random_real_solution(
            self._solution_size, self._lower_bounds, self._upper_bounds)

Response: Return a single random solution.


Instruction:
How would you implement a function in Python 3 that
makes a new population after each optimization iteration.

Response:
def next_population(self, population, fitnesses):
        """Make a new population after each optimization iteration.

        Args:
            population: The population current population of solutions.
            fitnesses: The fitness associated with each solution in the population
        Returns:
            list; a list of solutions.
        """
        return [self._next_solution() for _ in range(self._population_size)]


Instruction:
Explain what the following Python 3 code does
def _build_tree(self):

        groups = self._groups or self.get_children_paths(self.root_path)
        for group in groups:
            node = Node(name=group, parent=self.root)
            self.root.children.append(node)
            self._init_sub_groups(node)

Response: Build a full or partial tree based on the groups and sub - groups specified.


Instruction:
Implement a function in Python 3 to
initialise sub - groups and create any that do not already exist.

Response:
def _init_sub_groups(self, parent):
        """
        Initialise sub-groups, and create any that do not already exist.
        """

        if self._sub_groups:
            for sub_group in self._sub_groups:
                for component in split_path_components(sub_group):
                    fp = os.path.join(parent.full_path, component)
                    if os.path.exists(fp):
                        node = Node(name=component, parent=parent)
                        parent.children.append(node)
                    else:
                        node = parent.create_cgroup(component)
                    parent = node
                self._init_children(node)
        else:
            self._init_children(parent)


Instruction:
Explain what the following Python 3 code does
def _init_children(self, parent):

        for dir_name in self.get_children_paths(parent.full_path):
            child = Node(name=dir_name, parent=parent)
            parent.children.append(child)
            self._init_children(child)

Response: Initialize each node s children - essentially build the tree.


Instruction:
How would you explain what the following Python 3 function does
def full_path(self):

        if self.parent:
            return os.path.join(self.parent.full_path, self.name)
        return self.name

Response: Absolute system path to the node


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _get_node_type(self):

        if self.parent is None:
            return self.NODE_ROOT
        elif self.parent.node_type == self.NODE_ROOT:
            return self.NODE_CONTROLLER_ROOT
        elif b".slice" in self.name or b'.partition' in self.name:
            return self.NODE_SLICE
        elif b".scope" in self.name:
            return self.NODE_SCOPE
        else:
            return self.NODE_CGROUP

Response: Returns the current node s type


Instruction:
Explain what the following Python 3 code does
def _get_controller_type(self):

        if self.node_type == self.NODE_CONTROLLER_ROOT and self.name in self.CONTROLLERS:
            return self.name
        elif self.parent:
            return self.parent.controller_type
        else:
            return None

Response: Returns the current node s controller type


Instruction:
How would you implement a function in Python 3 that
creates a cgroup by name and attach it under this node.

Response:
def create_cgroup(self, name):
        """
        Create a cgroup by name and attach it under this node.
        """
        node = Node(name, parent=self)
        if node in self.children:
            raise RuntimeError('Node {} already exists under {}'.format(name, self.path))

        name = name.encode()
        fp = os.path.join(self.full_path, name)
        os.mkdir(fp)
        self.children.append(node)
        return node


Instruction:
Here you have a function in Python 3, explain what it does
def delete_cgroup(self, name):
        name = name.encode()
        fp = os.path.join(self.full_path, name)
        if os.path.exists(fp):
            os.rmdir(fp)
        node = Node(name, parent=self)
        try:
            self.children.remove(node)
        except ValueError:
            return

Response: Delete a cgroup by name and detach it from this node. Raises OSError if the cgroup is not empty.


Instruction:
Explain what the following Python 3 code does
def delete_empty_children(self):
        for child in self.children:
            child.delete_empty_children()
            try:
                if os.path.exists(child.full_path):
                    os.rmdir(child.full_path)
            except OSError: pass
            else: self.children.remove(child)

Response: Delete any empty children of this node.


Instruction:
How would you implement a function in Python 3 that
adds a node to the group.

Response:
def add_node(self, node):
        """
        A a Node object to the group. Only one node per cgroup is supported
        """
        if self.controllers.get(node.controller_type, None):
            raise RuntimeError("Cannot add node {} to the node group. A node for {} group is already assigned".format(
                node,
                node.controller_type
            ))
        self.nodes.append(node)
        if node.controller:
            self.controllers[node.controller_type] = node.controller
            setattr(self, node.controller_type, node.controller)


Instruction:
Explain what the following Python 3 code does
def group_tasks(self):
        tasks = set()
        for node in walk_tree(self):
            for ctrl in node.controllers.values():
                tasks.update(ctrl.tasks)
        return tasks

Response: All tasks in the hierarchy affected by this group.


Instruction:
How would you code a function in Python 3 to
return a set of all the tasks in this exact group.

Response:
def tasks(self):
        """Tasks in this exact group"""
        tasks = set()
        for ctrl in self.controllers.values():
            tasks.update(ctrl.tasks)
        return tasks


Instruction:
Can you tell what is the following Python 3 function doing
def filepath(self, filename):

        return os.path.join(self.node.full_path, filename)

Response: The full path to a file


Instruction:
Can you generate the documentation for the following Python 3 function
def get_property(self, filename):

        with open(self.filepath(filename)) as f:
            return f.read().strip()

Response: Opens the file and reads the value


Instruction:
Can you write a function in Python 3 where it
opens the file and writes the value

Response:
def set_property(self, filename, value):
        """Opens the file and writes the value"""

        with open(self.filepath(filename), "w") as f:
            return f.write(str(value))


Instruction:
Here you have a function in Python 3, explain what it does
def walk_tree(root):
    yield root

    for child in root.children:
        for el in walk_tree(child):
            yield el

Response: Walks the tree and returns an iterator over the elements.


Instruction:
How would you explain what the following Python 3 function does
def walk_up_tree(root):
    for child in root.children:
        for el in walk_up_tree(child):
            yield el

    yield root

Response: Recursively walks up the tree and returns an iterator over the elements in the tree.


Instruction:
Make a summary of the following Python 3 code
def get_device_major_minor(dev_path):
    stat = os.lstat(dev_path)
    return os.major(stat.st_rdev), os.minor(stat.st_rdev)

Response: Returns the device major and minor tuple for simplicity


Instruction:
Implement a function in Python 3 to
validate that this instance matches its schema.

Response:
def validate(self):
        """
        Validate that this instance matches its schema.

        """
        schema = Schema(self.__class__.SCHEMA)
        resolver = RefResolver.from_schema(
            schema,
            store=REGISTRY,
        )
        validate(self, schema, resolver=resolver)


Instruction:
Given the following Python 3 function, write the documentation
def dumps(self):
        with closing(StringIO()) as fileobj:
            self.dump(fileobj)
            return fileobj.getvalue()

Response: Dump this instance as YAML.


Instruction:
Can you generate the documentation for the following Python 3 function
def loads(cls, s):
        with closing(StringIO(s)) as fileobj:
            return cls.load(fileobj)

Response: Load an instance of this class from a string.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def property_schema(self, key):
        schema = self.__class__.SCHEMA

        # first try plain properties
        plain_schema = schema.get("properties", {}).get(key)
        if plain_schema is not None:
            return plain_schema
        # then try pattern properties
        pattern_properties = schema.get("patternProperties", {})
        for pattern, pattern_schema in pattern_properties.items():
            if match(pattern, key):
                return pattern_schema
        # finally try additional properties (defaults to true per JSON Schema)
        return schema.get("additionalProperties", True)

Response: Lookup the schema for a specific property.


Instruction:
Make a summary of the following Python 3 code
def make(class_name, base, schema):
    return type(class_name, (base,), dict(SCHEMA=schema))

Response: Create a new schema aware type.


Instruction:
Given the following Python 3 function, write the documentation
def make_definition(name, base, schema):
    class_name = make_class_name(name)
    cls = register(make(class_name, base, schema))
    globals()[class_name] = cls

Response: Create a new definition.


Instruction:
Explain what the following Python 3 code does
def register(cls):
    definition_name = make_definition_name(cls.__name__)
    REGISTRY[definition_name] = cls
    return cls

Response: Register a class as a new node in the registry.


Instruction:
Can you tell what is the following Python 3 function doing
def lookup(schema):
    if not isinstance(schema, dict) or "$ref" not in schema:
        return None

    ref = schema["$ref"]

    return REGISTRY.get(ref)

Response: Lookup a class by property schema.


Instruction:
Make a summary of the following Python 3 code
def connect(self):
    if self._connected or self.is_alive():
      raise ConnectionExistsError("Already connected")
    # After starting the thread we wait for it to post us
    # an event signifying that connection is established. This
    # ensures that the caller only resumes when we are fully connected.
    self.start()
    with self._lock:
      self._connect_cond.wait_for(lambda: self._connected)

Response: Connects to the lutron controller.


Instruction:
Write a Python 3 script for
sending the specified command to the lutron controller.

Response:
def _send_locked(self, cmd):
    """Sends the specified command to the lutron controller.

    Assumes self._lock is held.
    """
    _LOGGER.debug("Sending: %s" % cmd)
    try:
      self._telnet.write(cmd.encode('ascii') + b'\r\n')
    except BrokenPipeError:
      self._disconnect_locked()


Instruction:
Can you tell what is the following Python 3 function doing
def _do_login_locked(self):
    self._telnet = telnetlib.Telnet(self._host)
    self._telnet.read_until(LutronConnection.USER_PROMPT)
    self._telnet.write(self._user + b'\r\n')
    self._telnet.read_until(LutronConnection.PW_PROMPT)
    self._telnet.write(self._password + b'\r\n')
    self._telnet.read_until(LutronConnection.PROMPT)

    self._send_locked("#MONITORING,12,2")
    self._send_locked("#MONITORING,255,2")
    self._send_locked("#MONITORING,3,1")
    self._send_locked("#MONITORING,4,1")
    self._send_locked("#MONITORING,5,1")
    self._send_locked("#MONITORING,6,1")
    self._send_locked("#MONITORING,8,1")

Response:
Executes the login procedure and sets up some
    connection defaults like turning off the prompt etc.


Instruction:
Can you write a function in Python 3 where it
closes the current connection.

Response:
def _disconnect_locked(self):
    """Closes the current connection. Assume self._lock is held."""
    self._connected = False
    self._connect_cond.notify_all()
    self._telnet = None
    _LOGGER.warning("Disconnected")


Instruction:
Given the following Python 3 function, write the documentation
def _maybe_reconnect(self):
    with self._lock:
      if not self._connected:
        _LOGGER.info("Connecting")
        self._do_login_locked()
        self._connected = True
        self._connect_cond.notify_all()
        _LOGGER.info("Connected")

Response: Reconnects to the controller if we have previously disconnected.


Instruction:
Here you have a function in Python 3, explain what it does
def run(self):
    _LOGGER.info("Started")
    while True:
      self._maybe_reconnect()
      line = ''
      try:
        # If someone is sending a command, we can lose our connection so grab a
        # copy beforehand. We don't need the lock because if the connection is
        # open, we are the only ones that will read from telnet (the reconnect
        # code runs synchronously in this loop).
        t = self._telnet
        if t is not None:
          line = t.read_until(b"\n")
      except EOFError:
        try:
          self._lock.acquire()
          self._disconnect_locked()
          continue
        finally:
          self._lock.release()
      self._recv_cb(line.decode('ascii').rstrip())

Response: Main thread function to maintain connection and receive remote status.


Instruction:
How would you explain what the following Python 3 function does
def parse(self):
    import xml.etree.ElementTree as ET

    root = ET.fromstring(self._xml_db_str)
    # The structure is something like this:
    # <Areas>
    #   <Area ...>
    #     <DeviceGroups ...>
    #     <Scenes ...>
    #     <ShadeGroups ...>
    #     <Outputs ...>
    #     <Areas ...>
    #       <Area ...>

    # First area is useless, it's the top-level project area that defines the
    # "house". It contains the real nested Areas tree, which is the one we want.
    top_area = root.find('Areas').find('Area')
    self.project_name = top_area.get('Name')
    areas = top_area.find('Areas')
    for area_xml in areas.getiterator('Area'):
      area = self._parse_area(area_xml)
      self.areas.append(area)
    return True

Response:
Main entry point into the parser. It interprets and creates all the
    relevant Lutron objects and stuffs them into the appropriate hierarchy.


Instruction:
How would you explain what the following Python 3 function does
def _parse_area(self, area_xml):
    area = Area(self._lutron,
                name=area_xml.get('Name'),
                integration_id=int(area_xml.get('IntegrationID')),
                occupancy_group_id=area_xml.get('OccupancyGroupAssignedToID'))
    for output_xml in area_xml.find('Outputs'):
      output = self._parse_output(output_xml)
      area.add_output(output)
    # device group in our case means keypad
    # device_group.get('Name') is the location of the keypad
    for device_group in area_xml.find('DeviceGroups'):
      if device_group.tag == 'DeviceGroup':
        devs = device_group.find('Devices')
      elif device_group.tag == 'Device':
        devs = [device_group]
      else:
        _LOGGER.info("Unknown tag in DeviceGroups child %s" % devs)
        devs = []
      for device_xml in devs:
        if device_xml.tag != 'Device':
          continue
        if device_xml.get('DeviceType') in (
            'SEETOUCH_KEYPAD',
            'SEETOUCH_TABLETOP_KEYPAD',
            'PICO_KEYPAD',
            'HYBRID_SEETOUCH_KEYPAD',
            'MAIN_REPEATER'):
          keypad = self._parse_keypad(device_xml)
          area.add_keypad(keypad)
        elif device_xml.get('DeviceType') == 'MOTION_SENSOR':
          motion_sensor = self._parse_motion_sensor(device_xml)
          area.add_sensor(motion_sensor)
        #elif device_xml.get('DeviceType') == 'VISOR_CONTROL_RECEIVER':
    return area

Response: Parses an Area tag which is effectively a room depending on how the Lutron controller programming was done.


Instruction:
How would you explain what the following Python 3 function does
def _parse_output(self, output_xml):
    output = Output(self._lutron,
                    name=output_xml.get('Name'),
                    watts=int(output_xml.get('Wattage')),
                    output_type=output_xml.get('OutputType'),
                    integration_id=int(output_xml.get('IntegrationID')))
    return output

Response: Parses an output from a XML response.


Instruction:
How would you code a function in Python 3 to
parse a keypad device.

Response:
def _parse_keypad(self, keypad_xml):
    """Parses a keypad device (the Visor receiver is technically a keypad too)."""
    keypad = Keypad(self._lutron,
                    name=keypad_xml.get('Name'),
                    integration_id=int(keypad_xml.get('IntegrationID')))
    components = keypad_xml.find('Components')
    if not components:
      return keypad
    for comp in components:
      if comp.tag != 'Component':
        continue
      comp_type = comp.get('ComponentType')
      if comp_type == 'BUTTON':
        button = self._parse_button(keypad, comp)
        keypad.add_button(button)
      elif comp_type == 'LED':
        led = self._parse_led(keypad, comp)
        keypad.add_led(led)
    return keypad


Instruction:
Explain what the following Python 3 code does
def _parse_button(self, keypad, component_xml):
    button_xml = component_xml.find('Button')
    name = button_xml.get('Engraving')
    button_type = button_xml.get('ButtonType')
    direction = button_xml.get('Direction')
    # Hybrid keypads have dimmer buttons which have no engravings.
    if button_type == 'SingleSceneRaiseLower':
      name = 'Dimmer ' + direction
    if not name:
      name = "Unknown Button"
    button = Button(self._lutron, keypad,
                    name=name,
                    num=int(component_xml.get('ComponentNumber')),
                    button_type=button_type,
                    direction=direction)
    return button

Response: Parses a button device that part of a keypad.


Instruction:
Can you write a function in Python 3 where it
parses an LED device that part of a keypad.

Response:
def _parse_led(self, keypad, component_xml):
    """Parses an LED device that part of a keypad."""
    component_num = int(component_xml.get('ComponentNumber'))
    led_num = component_num - 80
    led = Led(self._lutron, keypad,
              name=('LED %d' % led_num),
              led_num=led_num,
              component_num=component_num)
    return led


Instruction:
Can you implement a function in Python 3 that
parses a motion sensor object.

Response:
def _parse_motion_sensor(self, sensor_xml):
    """Parses a motion sensor object.

    TODO: We don't actually do anything with these yet. There's a lot of info
    that needs to be managed to do this right. We'd have to manage the occupancy
    groups, what's assigned to them, and when they go (un)occupied. We'll handle
    this later.
    """
    return MotionSensor(self._lutron,
                        name=sensor_xml.get('Name'),
                        integration_id=int(sensor_xml.get('IntegrationID')))


Instruction:
How would you explain what the following Python 3 function does
def subscribe(self, obj, handler):
    if not isinstance(obj, LutronEntity):
      raise InvalidSubscription("Subscription target not a LutronEntity")
    _LOGGER.warning("DEPRECATED: Subscribing via Lutron.subscribe is obsolete. "
                    "Please use LutronEntity.subscribe")
    if obj not in self._legacy_subscribers:
      self._legacy_subscribers[obj] = handler
      obj.subscribe(self._dispatch_legacy_subscriber, None)

Response: Subscribes to status updates of the requested object.


Instruction:
Can you tell what is the following Python 3 function doing
def register_id(self, cmd_type, obj):
    ids = self._ids.setdefault(cmd_type, {})
    if obj.id in ids:
      raise IntegrationIdExistsError
    self._ids[cmd_type][obj.id] = obj

Response:
Registers an object through its integration id to receive update
    notifications.


Instruction:
Here you have a function in Python 3, explain what it does
def _dispatch_legacy_subscriber(self, obj, *args, **kwargs):
    if obj in self._legacy_subscribers:
      self._legacy_subscribers[obj](obj)

Response:
This dispatches the registered callback for obj. This is only used by legacy subscribers since new users should register with the target
    object directly.


Instruction:
Given the following Python 3 function, write the documentation
def _recv(self, line):
    if line == '':
      return
    # Only handle query response messages, which are also sent on remote status
    # updates (e.g. user manually pressed a keypad button)
    if line[0] != Lutron.OP_RESPONSE:
      _LOGGER.debug("ignoring %s" % line)
      return
    parts = line[1:].split(',')
    cmd_type = parts[0]
    integration_id = int(parts[1])
    args = parts[2:]
    if cmd_type not in self._ids:
      _LOGGER.info("Unknown cmd %s (%s)" % (cmd_type, line))
      return
    ids = self._ids[cmd_type]
    if integration_id not in ids:
      _LOGGER.warning("Unknown id %d (%s)" % (integration_id, line))
      return
    obj = ids[integration_id]
    handled = obj.handle_update(args)

Response: Invoked by the connection manager to process incoming data.


Instruction:
How would you implement a function in Python 3 that
formats and sends the requested command to the Lutron controller.

Response:
def send(self, op, cmd, integration_id, *args):
    """Formats and sends the requested command to the Lutron controller."""
    out_cmd = ",".join(
        (cmd, str(integration_id)) + tuple((str(x) for x in args)))
    self._conn.send(op + out_cmd)


Instruction:
Given the following Python 3 function, write the documentation
def load_xml_db(self):

    import urllib.request
    xmlfile = urllib.request.urlopen('http://' + self._host + '/DbXmlInfo.xml')
    xml_db = xmlfile.read()
    xmlfile.close()
    _LOGGER.info("Loaded xml db")

    parser = LutronXmlDbParser(lutron=self, xml_db_str=xml_db)
    assert(parser.parse())     # throw our own exception
    self._areas = parser.areas
    self._name = parser.project_name

    _LOGGER.info('Found Lutron project: %s, %d areas' % (
        self._name, len(self.areas)))

    return True

Response: Load the Lutron database from the server.


Instruction:
Can you tell what is the following Python 3 function doing
def request(self, action):
    ev = threading.Event()
    first = False
    with self.__lock:
      if len(self.__events) == 0:
        first = True
      self.__events.append(ev)
    if first:
      action()
    return ev

Response: Request an action to be performed in case one.


Instruction:
Make a summary of the following Python 3 code
def _dispatch_event(self, event: LutronEvent, params: Dict):
    for handler, context in self._subscribers:
      handler(self, context, event, params)

Response: Dispatches the specified event to all the subscribers.


Instruction:
Write a Python 3 script to
subscribe to events from this entity.

Response:
def subscribe(self, handler: LutronEventHandler, context):
    """Subscribes to events from this entity.

    handler: A callable object that takes the following arguments (in order)
             obj: the LutrongEntity object that generated the event
             context: user-supplied (to subscribe()) context object
             event: the LutronEvent that was generated.
             params: a dict of event-specific parameters

    context: User-supplied, opaque object that will be passed to handler.
    """
    self._subscribers.append((handler, context))


Instruction:
How would you code a function in Python 3 to
handle an update event for this object.

Response:
def handle_update(self, args):
    """Handles an event update for this object, e.g. dimmer level change."""
    _LOGGER.debug("handle_update %d -- %s" % (self._integration_id, args))
    state = int(args[0])
    if state != Output._ACTION_ZONE_LEVEL:
      return False
    level = float(args[1])
    _LOGGER.debug("Updating %d(%s): s=%d l=%f" % (
        self._integration_id, self._name, state, level))
    self._level = level
    self._query_waiters.notify()
    self._dispatch_event(Output.Event.LEVEL_CHANGED, {'level': self._level})
    return True


Instruction:
How would you implement a function in Python 3 that
returns the current output level by querying the remote controller.

Response:
def level(self):
    """Returns the current output level by querying the remote controller."""
    ev = self._query_waiters.request(self.__do_query_level)
    ev.wait(1.0)
    return self._level


Instruction:
How would you implement a function in Python 3 that
sets the new output level.

Response:
def level(self, new_level):
    """Sets the new output level."""
    if self._level == new_level:
      return
    self._lutron.send(Lutron.OP_EXECUTE, Output._CMD_TYPE, self._integration_id,
        Output._ACTION_ZONE_LEVEL, "%.2f" % new_level)
    self._level = new_level


Instruction:
Write a Python 3 script for
handling the specified action on this component.

Response:
def handle_update(self, action, params):
    """Handle the specified action on this component."""
    _LOGGER.debug('Keypad: "%s" Handling "%s" Action: %s Params: %s"' % (
                  self._keypad.name, self.name, action, params))
    return False


Instruction:
Here you have a function in Python 3, explain what it does
def press(self):
    self._lutron.send(Lutron.OP_EXECUTE, Keypad._CMD_TYPE, self._keypad.id,
                      self.component_number, Button._ACTION_PRESS)

Response: Triggers a simulated button press to the Keypad.


Instruction:
Can you write a function in Python 3 where it
handles the specified action on this keypad.

Response:
def handle_update(self, action, params):
    """Handle the specified action on this component."""
    _LOGGER.debug('Keypad: "%s" %s Action: %s Params: %s"' % (
                  self._keypad.name, self, action, params))
    ev_map = {
        Button._ACTION_PRESS: Button.Event.PRESSED,
        Button._ACTION_RELEASE: Button.Event.RELEASED
    }
    if action not in ev_map:
      _LOGGER.debug("Unknown action %d for button %d in keypad %d" % (
          action, self.number, self.keypad.name))
      return False
    self._dispatch_event(ev_map[action], {})
    return True


Instruction:
Can you tell what is the following Python 3 function doing
def __do_query_state(self):
    self._lutron.send(Lutron.OP_QUERY, Keypad._CMD_TYPE, self._keypad.id,
            self.component_number, Led._ACTION_LED_STATE)

Response: Helper method to perform the actual query for the current LED state.


Instruction:
How would you code a function in Python 3 to
return the current LED state by querying the remote controller.

Response:
def state(self):
    """Returns the current LED state by querying the remote controller."""
    ev = self._query_waiters.request(self.__do_query_state)
    ev.wait(1.0)
    return self._state


Instruction:
Write a Python 3 script for
setting the new led state.

Response:
def state(self, new_state: bool):
    """Sets the new led state.

    new_state: bool
    """
    self._lutron.send(Lutron.OP_EXECUTE, Keypad._CMD_TYPE, self._keypad.id,
                      self.component_number, Led._ACTION_LED_STATE,
                      int(new_state))
    self._state = new_state


Instruction:
Can you generate a brief explanation for the following Python 3 code
def handle_update(self, action, params):
    _LOGGER.debug('Keypad: "%s" %s Action: %s Params: %s"' % (
                  self._keypad.name, self, action, params))
    if action != Led._ACTION_LED_STATE:
      _LOGGER.debug("Unknown action %d for led %d in keypad %d" % (
          action, self.number, self.keypad.name))
      return False
    elif len(params) < 1:
      _LOGGER.debug("Unknown params %s (action %d on led %d in keypad %d)" % (
          params, action, self.number, self.keypad.name))
      return False
    self._state = bool(params[0])
    self._query_waiters.notify()
    self._dispatch_event(Led.Event.STATE_CHANGED, {'state': self._state})
    return True

Response: Handle the specified action on this component.


Instruction:
Given the following Python 3 function, write the documentation
def add_button(self, button):
    self._buttons.append(button)
    self._components[button.component_number] = button

Response:
Adds a button that s part of this keypad. We ll use this to
    dispatch button events.


Instruction:
Implement a function in Python 3 to
add an LED that s part of this keypad.

Response:
def add_led(self, led):
    """Add an LED that's part of this keypad."""
    self._leds.append(led)
    self._components[led.component_number] = led


Instruction:
Create a Python 3 function to
validate a pattern against the STIX Pattern grammar.

Response:
def run_validator(pattern):
    """
    Validates a pattern against the STIX Pattern grammar.  Error messages are
    returned in a list.  The test passed if the returned list is empty.
    """

    start = ''
    if isinstance(pattern, six.string_types):
        start = pattern[:2]
        pattern = InputStream(pattern)

    if not start:
        start = pattern.readline()[:2]
        pattern.seek(0)

    parseErrListener = STIXPatternErrorListener()

    lexer = STIXPatternLexer(pattern)
    # it always adds a console listener by default... remove it.
    lexer.removeErrorListeners()

    stream = CommonTokenStream(lexer)

    parser = STIXPatternParser(stream)
    parser.buildParseTrees = False
    # it always adds a console listener by default... remove it.
    parser.removeErrorListeners()
    parser.addErrorListener(parseErrListener)

    # To improve error messages, replace "<INVALID>" in the literal
    # names with symbolic names.  This is a hack, but seemed like
    # the simplest workaround.
    for i, lit_name in enumerate(parser.literalNames):
        if lit_name == u"<INVALID>":
            parser.literalNames[i] = parser.symbolicNames[i]

    parser.pattern()

    # replace with easier-to-understand error message
    if not (start[0] == '[' or start == '(['):
        parseErrListener.err_strings[0] = "FAIL: Error found at line 1:0. " \
                                          "input is missing square brackets"

    return parseErrListener.err_strings


Instruction:
How would you code a function in Python 3 to
validate a user input string.

Response:
def validate(user_input, ret_errs=False, print_errs=False):
    """
    Wrapper for run_validator function that returns True if the user_input
    contains a valid STIX pattern or False otherwise. The error messages may
    also be returned or printed based upon the ret_errs and print_errs arg
    values.
    """

    errs = run_validator(user_input)
    passed = len(errs) == 0

    if print_errs:
        for err in errs:
            print(err)

    if ret_errs:
        return passed, errs

    return passed


Instruction:
Can you implement a function in Python 3 that
continues to validate patterns until it encounters EOF within a pattern file or Ctrl-C is pressed by the user.

Response:
def main():
    """
    Continues to validate patterns until it encounters EOF within a pattern
    file or Ctrl-C is pressed by the user.
    """
    parser = argparse.ArgumentParser(description='Validate STIX Patterns.')
    parser.add_argument('-f', '--file',
                        help="Specify this arg to read patterns from a file.",
                        type=argparse.FileType("r"))
    args = parser.parse_args()

    pass_count = fail_count = 0

    # I tried using a generator (where each iteration would run raw_input()),
    # but raw_input()'s behavior seems to change when called from within a
    # generator: I only get one line, then the generator completes!  I don't
    # know why behavior changes...
    import functools
    if args.file:
        nextpattern = args.file.readline
    else:
        nextpattern = functools.partial(six.moves.input, "Enter a pattern to validate: ")

    try:
        while True:
            pattern = nextpattern()
            if not pattern:
                break
            tests_passed, err_strings = validate(pattern, True)

            if tests_passed:
                print("\nPASS: %s" % pattern)
                pass_count += 1

            else:
                for err in err_strings:
                    print(err, '\n')
                fail_count += 1
    except (EOFError, KeyboardInterrupt):
        pass
    finally:
        if args.file:
            args.file.close()

    print("\nPASSED:", pass_count, " patterns")
    print("FAILED:", fail_count, " patterns")


Instruction:
Write a Python 3 script to
convert a StringLiteral token to a plain string.

Response:
def _string_literal_to_string(string_literal_token):
    """Converts the StringLiteral token to a plain string: get text content,
    removes quote characters, and unescapes it.

    :param string_literal_token: The string literal
    :return:
    """
    token_text = string_literal_token.getText()
    return token_text[1:-1].replace(u"\\'", u"'"). \
        replace(u"\\\\", u"\\")


Instruction:
Make a summary of the following Python 3 code
def get(self, request, *args, **kwargs):
        if 'html' in request.GET:
            # Output HTML
            content = self.render_html(*args, **kwargs)
            return HttpResponse(content)

        else:
            # Output PDF
            content = self.render_pdf(*args, **kwargs)

            response = HttpResponse(content, content_type='application/pdf')

            if (not self.inline or 'download' in request.GET) and 'inline' not in request.GET:
                response['Content-Disposition'] = 'attachment; filename=%s' % self.get_filename()

            response['Content-Length'] = len(content)

            return response

Response: Returns a HTTPResponse either of a PDF file HTML or a PDF file.


Instruction:
Can you generate the documentation for the following Python 3 function
def render_pdf(self, *args, **kwargs):
        html = self.render_html(*args, **kwargs)

        options = self.get_pdfkit_options()
        if 'debug' in self.request.GET and settings.DEBUG:
            options['debug-javascript'] = 1

        kwargs = {}
        wkhtmltopdf_bin = os.environ.get('WKHTMLTOPDF_BIN')
        if wkhtmltopdf_bin:
            kwargs['configuration'] = pdfkit.configuration(wkhtmltopdf=wkhtmltopdf_bin)

        pdf = pdfkit.from_string(html, False, options, **kwargs)

        return pdf

Response: Render the PDF and returns as bytes.


Instruction:
Here you have a function in Python 3, explain what it does
def get_filename(self):
        if self.filename is None:
            name = splitext(basename(self.template_name))[0]
            return '{}.pdf'.format(name)

        return self.filename

Response: Return the filename of the current object.


Instruction:
Implement a function in Python 3 to
render the template. :rtype: str

Response:
def render_html(self, *args, **kwargs):
        """
        Renders the template.

        :rtype: str
        """
        static_url = '%s://%s%s' % (self.request.scheme, self.request.get_host(), settings.STATIC_URL)
        media_url = '%s://%s%s' % (self.request.scheme, self.request.get_host(), settings.MEDIA_URL)

        with override_settings(STATIC_URL=static_url, MEDIA_URL=media_url):
            template = loader.get_template(self.template_name)
            context = self.get_context_data(*args, **kwargs)
            html = template.render(context)
            return html


Instruction:
How would you code a function in Python 3 to
inspect a pattern. This gives information regarding the sorts of operations content etc in use in the pattern.

Response:
def inspect(self):
        """
        Inspect a pattern.  This gives information regarding the sorts of
        operations, content, etc in use in the pattern.

        :return: Pattern information
        """

        inspector = stix2patterns.inspector.InspectionListener()
        self.walk(inspector)

        return inspector.pattern_data()


Instruction:
Write a Python 3 script to
walk the parse tree using the given listener.

Response:
def walk(self, listener):
        """Walk the parse tree, using the given listener.  The listener
        should be a
        stix2patterns.grammars.STIXPatternListener.STIXPatternListener (or
        subclass) instance."""
        antlr4.ParseTreeWalker.DEFAULT.walk(listener, self.__parse_tree)


Instruction:
Can you generate the documentation for the following Python 3 function
def __do_parse(self, pattern_str):
        in_ = antlr4.InputStream(pattern_str)
        lexer = STIXPatternLexer(in_)
        lexer.removeErrorListeners()  # remove the default "console" listener
        token_stream = antlr4.CommonTokenStream(lexer)

        parser = STIXPatternParser(token_stream)
        parser.removeErrorListeners()  # remove the default "console" listener
        error_listener = ParserErrorListener()
        parser.addErrorListener(error_listener)

        # I found no public API for this...
        # The default error handler tries to keep parsing, and I don't
        # think that's appropriate here.  (These error handlers are only for
        # handling the built-in RecognitionException errors.)
        parser._errHandler = antlr4.BailErrorStrategy()

        # To improve error messages, replace "<INVALID>" in the literal
        # names with symbolic names.  This is a hack, but seemed like
        # the simplest workaround.
        for i, lit_name in enumerate(parser.literalNames):
            if lit_name == u"<INVALID>":
                parser.literalNames[i] = parser.symbolicNames[i]

        # parser.setTrace(True)

        try:
            tree = parser.pattern()
            # print(tree.toStringTree(recog=parser))

            return tree
        except antlr4.error.Errors.ParseCancellationException as e:
            # The cancellation exception wraps the real RecognitionException
            # which caused the parser to bail.
            real_exc = e.args[0]

            # I want to bail when the first error is hit.  But I also want
            # a decent error message.  When an error is encountered in
            # Parser.match(), the BailErrorStrategy produces the
            # ParseCancellationException.  It is not a subclass of
            # RecognitionException, so none of the 'except' clauses which would
            # normally report an error are invoked.
            #
            # Error message creation is buried in the ErrorStrategy, and I can
            # (ab)use the API to get a message: register an error listener with
            # the parser, force an error report, then get the message out of the
            # listener.  Error listener registration is above; now we force its
            # invocation.  Wish this could be cleaner...
            parser._errHandler.reportError(parser, real_exc)

            # should probably chain exceptions if we can...
            # Should I report the cancellation or recognition exception as the
            # cause...?
            six.raise_from(ParseException(error_listener.error_message),
                           real_exc)

Response: Parses the given pattern and returns the parse tree.


Instruction:
Can you tell what is the following Python 3 function doing
def stdout(self):
        streams = self._payload.get('streams', None)
        return streams[0] if streams is not None and len(streams) >= 1 else ''

Response: Returns the stdout of the job or None if the job is not running.


Instruction:
Can you tell what is the following Python 3 function doing
def stderr(self):
        streams = self._payload.get('streams', None)
        return streams[1] if streams is not None and len(streams) >= 2 else ''

Response: Returns the stderr of the job


Instruction:
Create a Python 3 function for
returning true if the job result exists

Response:
def exists(self):
        """
        Returns true if the job is still running or zero-os still knows about this job ID

        After a job is finished, a job remains on zero-os for max of 5min where you still can read the job result
        after the 5 min is gone, the job result is no more fetchable
        :return: bool
        """
        r = self._client._redis
        flag = '{}:flag'.format(self._queue)
        return bool(r.exists(flag))


Instruction:
Write a Python 3 script for
returning True if the job is in running state False otherwise.

Response:
def running(self):
        """
        Returns true if job still in running state
        :return:
        """
        r = self._client._redis
        flag = '{}:flag'.format(self._queue)
        if bool(r.exists(flag)):
            return r.ttl(flag) is None

        return False


Instruction:
Can you write a function in Python 3 where it
streams the log messages from the process.

Response:
def stream(self, callback=None):
        """
        Runtime copy of job messages. This required the 'stream` flag to be set to True otherwise it will
        not be able to copy any output, while it will block until the process exits.

        :note: This function will block until it reaches end of stream or the process is no longer running.

        :param callback: callback method that will get called for each received message
                         callback accepts 3 arguments
                         - level int: the log message levels, refer to the docs for available levels
                                      and their meanings
                         - message str: the actual output message
                         - flags int: flags associated with this message
                                      - 0x2 means EOF with success exit status
                                      - 0x4 means EOF with error

                                      for example (eof = flag & 0x6) eof will be true for last message u will ever
                                      receive on this callback.

                         Note: if callback is none, a default callback will be used that prints output on stdout/stderr
                         based on level.
        :return: None
        """
        if callback is None:
            callback = Response.__default

        if not callable(callback):
            raise Exception('callback must be callable')

        queue = 'stream:%s' % self.id
        r = self._client._redis

        # we can terminate quickly by checking if the process is not running and it has no queued output.
        # if not self.running and r.llen(queue) == 0:
        #     return

        while True:
            data = r.blpop(queue, 10)
            if data is None:
                if not self.running:
                    break
                continue
            _, body = data
            payload = json.loads(body.decode())
            message = payload['message']
            line = message['message']
            meta = message['meta']
            callback(meta >> 16, line, meta & 0xff)

            if meta & 0x6 != 0:
                break


Instruction:
Can you create a Python 3 function that
waits for a job to finish and return the result.

Response:
def get(self, timeout=None):
        """
        Waits for a job to finish (max of given timeout seconds) and return job results. When a job exits get() will
        keep returning the same result until zero-os doesn't remember the job anymore (self.exists == False)

        :notes: the timeout here is a client side timeout, it's different than the timeout given to the job on start
        (like in system method) witch will cause the job to be killed if it exceeded this timeout.

        :param timeout: max time to wait for the job to finish in seconds
        :return: Return object
        """
        if timeout is None:
            timeout = self._client.timeout
        r = self._client._redis
        start = time.time()
        maxwait = timeout
        while maxwait > 0:
            if not self.exists:
                raise JobNotFoundError(self.id)
            v = r.brpoplpush(self._queue, self._queue, min(maxwait, 10))
            if v is not None:
                payload = json.loads(v.decode())
                r = Return(payload)
                logger.debug('%s << %s, stdout="%s", stderr="%s", data="%s"',
                             self._id, r.state, r.stdout, r.stderr, r.data[:1000])
                return r
            logger.debug('%s still waiting (%ss)', self._id, int(time.time() - start))
            maxwait -= 10
        raise TimeoutError()


Instruction:
How would you code a function in Python 3 to
get response as json

Response:
def get(self, timeout=None):
        """
        Get response as json, will fail if the job doesn't return a valid json response

        :param timeout: client side timeout in seconds
        :return: int
        """
        result = super().get(timeout)
        if result.state != 'SUCCESS':
            raise ResultError(result.data, result.code)
        if result.level != 20:
            raise ResultError('not a json response: %d' % result.level, 406)

        return json.loads(result.data)


Instruction:
Write a Python 3 script for
listing all running jobs

Response:
def list(self, id=None):
        """
        List all running jobs

        :param id: optional ID for the job to list
        """
        args = {'id': id}
        self._job_chk.check(args)
        return self._client.json('job.list', args)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def kill(self, id, signal=signal.SIGTERM):
        args = {
            'id': id,
            'signal': int(signal),
        }
        self._kill_chk.check(args)
        return self._client.json('job.kill', args)

Response: Kill a job with given id


Instruction:
Can you generate the documentation for the following Python 3 function
def list(self, id=None):
        args = {'pid': id}
        self._process_chk.check(args)
        return self._client.json('process.list', args)

Response: List all running processes


Instruction:
Make a summary of the following Python 3 code
def open(self, file, mode='r', perm=0o0644):
        args = {
            'file': file,
            'mode': mode,
            'perm': perm,
        }

        return self._client.json('filesystem.open', args)

Response: Opens a file on the node with the given path.


Instruction:
Write a Python 3 script to
move a path to a destination

Response:
def move(self, path, destination):
        """
        Move a path to destination

        :param path: source
        :param destination: destination
        :return:
        """
        args = {
            'path': path,
            'destination': destination,
        }

        return self._client.json('filesystem.move', args)


Instruction:
Can you tell what is the following Python 3 function doing
def chmod(self, path, mode, recursive=False):
        args = {
            'path': path,
            'mode': mode,
            'recursive': recursive,
        }

        return self._client.json('filesystem.chmod', args)

Response: Change file permissions of a file or directory.


Instruction:
Implement a Python 3 function for
changing file owner of path with user and group.

Response:
def chown(self, path, user, group, recursive=False):
        """
        Change file/dir owner

        :param path: path of file/dir
        :param user: user name
        :param group: group name
        :param recursive: apply chown recursively
        :return:
        """
        args = {
            'path': path,
            'user': user,
            'group': group,
            'recursive': recursive,
        }

        return self._client.json('filesystem.chown', args)


Instruction:
Explain what the following Python 3 code does
def read(self, fd):
        args = {
            'fd': fd,
        }

        data = self._client.json('filesystem.read', args)
        return base64.decodebytes(data.encode())

Response: Reads a block from the given file descriptor.


Instruction:
Write a Python 3 function for
writing a block of bytes to an open file descriptor.

Response:
def write(self, fd, bytes):
        """
        Write a block of bytes to an open file descriptor (that is open with one of the writing modes

        :param fd: file descriptor
        :param bytes: bytes block to write
        :return:

        :note: don't overkill the node with large byte chunks, also for large file upload check the upload method.
        """
        args = {
            'fd': fd,
            'block': base64.encodebytes(bytes).decode(),
        }

        return self._client.json('filesystem.write', args)


Instruction:
Explain what the following Python 3 code does
def upload(self, remote, reader):

        fd = self.open(remote, 'w')
        while True:
            chunk = reader.read(512 * 1024)
            if chunk == b'':
                break
            self.write(fd, chunk)
        self.close(fd)

Response: Uploads a file to the local cache.


Instruction:
Make a summary of the following Python 3 code
def download(self, remote, writer):

        fd = self.open(remote)
        while True:
            chunk = self.read(fd)
            if chunk == b'':
                break
            writer.write(chunk)
        self.close(fd)

Response: Downloads a file from the local file system.


Instruction:
Here you have a function in Python 3, explain what it does
def upload_file(self, remote, local):
        file = open(local, 'rb')
        try:
            self.upload(remote, file)
        finally:
            file.close()

Response: Uploads a file to the naculiac


Instruction:
Here you have a function in Python 3, explain what it does
def download_file(self, remote, local):
        file = open(local, 'wb')
        try:
            self.download(remote, file)
        finally:
            file.close()

Response: Downloads a file from remote to local.


Instruction:
Create a Python 3 function to
implement the low level command call, this needs to build the command structure and push it on the correct queue. :param command: Command name to execute supported by the node (ex: core.system, info.cpu, etc...) check documentation for list of built in commands :param arguments: A dict of required command arguments depends on the command name. :param queue: command queue (commands on the same queue are executed sequentially) :param max_time: kill job server side if it exceeded this amount of seconds :param stream: If True, process stdout and stderr are pushed to a special queue (stream:<id>) so client can stream output :param tags: job tags :param id: job id. Generated if not supplied :return: Response object

Response:
def raw(self, command, arguments, queue=None, max_time=None, stream=False, tags=None, id=None):
        """
        Implements the low level command call, this needs to build the command structure
        and push it on the correct queue.

        :param command: Command name to execute supported by the node (ex: core.system, info.cpu, etc...)
                        check documentation for list of built in commands
        :param arguments: A dict of required command arguments depends on the command name.
        :param queue: command queue (commands on the same queue are executed sequentially)
        :param max_time: kill job server side if it exceeded this amount of seconds
        :param stream: If True, process stdout and stderr are pushed to a special queue (stream:<id>) so
            client can stream output
        :param tags: job tags
        :param id: job id. Generated if not supplied
        :return: Response object
        """
        raise NotImplemented()


Instruction:
Here you have a function in Python 3, explain what it does
def sync(self, command, arguments, tags=None, id=None):
        response = self.raw(command, arguments, tags=tags, id=id)

        result = response.get()
        if result.state != 'SUCCESS':
            raise ResultError(msg='%s' % result.data, code=result.code)

        return result

Response: This method is used to sync the job with the server.


Instruction:
Can you create a Python 3 function that
executes a command and return the output of the command.

Response:
def system(self, command, dir='', stdin='', env=None, queue=None, max_time=None, stream=False, tags=None, id=None):
        """
        Execute a command

        :param command:  command to execute (with its arguments) ex: `ls -l /root`
        :param dir: CWD of command
        :param stdin: Stdin data to feed to the command stdin
        :param env: dict with ENV variables that will be exported to the command
        :param id: job id. Auto generated if not defined.
        :return:
        """
        parts = shlex.split(command)
        if len(parts) == 0:
            raise ValueError('invalid command')

        args = {
            'name': parts[0],
            'args': parts[1:],
            'dir': dir,
            'stdin': stdin,
            'env': env,
        }

        self._system_chk.check(args)
        response = self.raw(command='core.system', arguments=args,
                            queue=queue, max_time=max_time, stream=stream, tags=tags, id=id)

        return response


Instruction:
Explain what the following Python 3 code does
def bash(self, script, stdin='', queue=None, max_time=None, stream=False, tags=None, id=None):
        args = {
            'script': script,
            'stdin': stdin,
        }
        self._bash_chk.check(args)
        response = self.raw(command='bash', arguments=args,
                            queue=queue, max_time=max_time, stream=stream, tags=tags, id=id)

        return response

Response: Execute a bash script or run a process inside a bash shell.


Instruction:
Can you generate the documentation for the following Python 3 function
def subscribe(self, job, id=None):
        return self.raw('core.subscribe', {'id': job}, stream=True, id=id)

Response: Subscribes to the job logs.


Instruction:
Write a Python 3 script to
implement the low level command call, this needs to build the command structure and push it on the correct queue. :param command: Command name to execute supported by the node (ex: core.system, info.cpu, etc...) check documentation for list of built in commands :param arguments: A dict of required command arguments depends on the command name. :param queue: command queue (commands on the same queue are executed sequentially) :param max_time: kill job server side if it exceeded this amount of seconds :param stream: If True, process stdout and stderr are pushed to a special queue (stream:<id>) so client can stream output :param tags: job tags :param id: job id. Generated if not supplied :return: Response object

Response:
def raw(self, command, arguments, queue=None, max_time=None, stream=False, tags=None, id=None):
        """
        Implements the low level command call, this needs to build the command structure
        and push it on the correct queue.

        :param command: Command name to execute supported by the node (ex: core.system, info.cpu, etc...)
                        check documentation for list of built in commands
        :param arguments: A dict of required command arguments depends on the command name.
        :param queue: command queue (commands on the same queue are executed sequentially)
        :param max_time: kill job server side if it exceeded this amount of seconds
        :param stream: If True, process stdout and stderr are pushed to a special queue (stream:<id>) so
            client can stream output
        :param tags: job tags
        :param id: job id. Generated if not supplied
        :return: Response object
        """
        args = {
            'container': self._container,
            'command': {
                'command': command,
                'arguments': arguments,
                'queue': queue,
                'max_time': max_time,
                'stream': stream,
                'tags': tags,
                'id': id,
            },
        }

        # check input
        self._raw_chk.check(args)

        response = self._client.raw('corex.dispatch', args)

        result = response.get()
        if result.state != 'SUCCESS':
            raise RuntimeError('failed to dispatch command to container: %s' % result.data)

        cmd_id = json.loads(result.data)
        return self._client.response_for(cmd_id)