Instruction:
Create a Python 3 function for
performing an optimization step.

Response:
def minimize(self, time, variables, **kwargs):
        """
        Performs an optimization step.

        Args:
            time: Time tensor.
            variables: List of variables to optimize.
            **kwargs: Additional optimizer-specific arguments. The following arguments are used
                by some optimizers:
            - arguments: Dict of arguments for callables, like fn_loss.
            - fn_loss: A callable returning the loss of the current model.
            - fn_reference: A callable returning the reference values, in case of a comparative
                loss.
            - fn_kl_divergence: A callable returning the KL-divergence relative to the
                current model.
            - sampled_loss: A sampled loss (integer).
            - return_estimated_improvement: Returns the estimated improvement resulting from
                the natural gradient calculation if true.
            - source_variables: List of source variables to synchronize with.
            - global_variables: List of global variables to apply the proposed optimization
                step to.

        Returns:
            The optimization operation.
        """
        # # Add training variable gradient histograms/scalars to summary output
        # # if 'gradients' in self.summary_labels:
        # if any(k in self.summary_labels for k in ['gradients', 'gradients_histogram', 'gradients_scalar']):
        #     valid = True
        #     if isinstance(self, tensorforce.core.optimizers.TFOptimizer):
        #         gradients = self.optimizer.compute_gradients(kwargs['fn_loss']())
        #     elif isinstance(self.optimizer, tensorforce.core.optimizers.TFOptimizer):
        #         # This section handles "Multi_step" and may handle others
        #         # if failure is found, add another elif to handle that case
        #         gradients = self.optimizer.optimizer.compute_gradients(kwargs['fn_loss']())
        #     else:
        #         # Didn't find proper gradient information
        #         valid = False

        #     # Valid gradient data found, create summary data items
        #     if valid:
        #         for grad, var in gradients:
        #             if grad is not None:
        #                 if any(k in self.summary_labels for k in ('gradients', 'gradients_scalar')):
        #                     axes = list(range(len(grad.shape)))
        #                     mean, var = tf.nn.moments(grad, axes)
        #                     tf.contrib.summary.scalar(name='gradients/' + var.name + "/mean", tensor=mean)
        #                     tf.contrib.summary.scalar(name='gradients/' + var.name + "/variance", tensor=var)
        #                 if any(k in self.summary_labels for k in ('gradients', 'gradients_histogram')):
        #                     tf.contrib.summary.histogram(name='gradients/' + var.name, tensor=grad)

        deltas = self.step(time=time, variables=variables, **kwargs)
        with tf.control_dependencies(control_inputs=deltas):
            return tf.no_op()


Instruction:
Given the following Python 3 function, write the documentation
def from_spec(spec, kwargs=None):
        optimizer = util.get_object(
            obj=spec,
            predefined_objects=tensorforce.core.optimizers.optimizers,
            kwargs=kwargs
        )
        assert isinstance(optimizer, Optimizer)
        return optimizer

Response: Creates an optimizer from a dictionary.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def np_dtype(dtype):
    if dtype == 'float' or dtype == float or dtype == np.float32 or dtype == tf.float32:
        return np.float32
    elif dtype == np.float64 or dtype == tf.float64:
        return np.float64
    elif dtype == np.float16 or dtype == tf.float16:
        return np.float16
    elif dtype == 'int' or dtype == int or dtype == np.int32 or dtype == tf.int32:
        return np.int32
    elif dtype == np.int64 or dtype == tf.int64:
        return np.int64
    elif dtype == np.int16 or dtype == tf.int16:
        return np.int16
    elif dtype == 'bool' or dtype == bool or dtype == np.bool_ or dtype == tf.bool:
        return np.bool_
    else:
        raise TensorForceError("Error: Type conversion from type {} not supported.".format(str(dtype)))

Response: Translates dtype specifications in configurations to Numpy data types.


Instruction:
Can you tell what is the following Python 3 function doing
def get_tensor_dependencies(tensor):
    dependencies = set()
    dependencies.update(tensor.op.inputs)
    for sub_op in tensor.op.inputs:
        dependencies.update(get_tensor_dependencies(sub_op))
    return dependencies

Response: Utility method to get all dependencies of a tensor.


Instruction:
Make a summary of the following Python 3 code
def get_object(obj, predefined_objects=None, default_object=None, kwargs=None):
    args = ()
    kwargs = dict() if kwargs is None else kwargs

    if isinstance(obj, str) and os.path.isfile(obj):
        with open(obj, 'r') as fp:
            obj = json.load(fp=fp)
    if isinstance(obj, dict):
        kwargs.update(obj)
        obj = kwargs.pop('type', None)

    if predefined_objects is not None and obj in predefined_objects:
        obj = predefined_objects[obj]
    elif isinstance(obj, str):
        if obj.find('.') != -1:
            module_name, function_name = obj.rsplit('.', 1)
            module = importlib.import_module(module_name)
            obj = getattr(module, function_name)
        else:
            raise TensorForceError("Error: object {} not found in predefined objects: {}".format(
                obj,
                list(predefined_objects or ())
            ))
    elif callable(obj):
        pass
    elif default_object is not None:
        args = (obj,)
        obj = default_object
    else:
        # assumes the object is already instantiated
        return obj

    return obj(*args, **kwargs)

Response: Utility method to map some kind of object specification to its content.


Instruction:
Can you create a Python 3 function that
registers the saver operations to the graph in context.

Response:
def register_saver_ops(self):
        """
        Registers the saver operations to the graph in context.
        """

        variables = self.get_savable_variables()
        if variables is None or len(variables) == 0:
            self._saver = None
            return

        base_scope = self._get_base_variable_scope()
        variables_map = {strip_name_scope(v.name, base_scope): v for v in variables}

        self._saver = tf.train.Saver(
            var_list=variables_map,
            reshape=False,
            sharded=False,
            max_to_keep=5,
            keep_checkpoint_every_n_hours=10000.0,
            name=None,
            restore_sequentially=False,
            saver_def=None,
            builder=None,
            defer_build=False,
            allow_empty=True,
            write_version=tf.train.SaverDef.V2,
            pad_step_number=False,
            save_relative_paths=True
        )


Instruction:
Create a Python 3 function for
saving the current state of the component.

Response:
def save(self, sess, save_path, timestep=None):
        """
        Saves this component's managed variables.

        Args:
            sess: The session for which to save the managed variables.
            save_path: The path to save data to.
            timestep: Optional, the timestep to append to the file name.

        Returns:
            Checkpoint path where the model was saved.
        """

        if self._saver is None:
            raise TensorForceError("register_saver_ops should be called before save")
        return self._saver.save(
            sess=sess,
            save_path=save_path,
            global_step=timestep,
            write_meta_graph=False,
            write_state=True,  # Do we need this?
        )


Instruction:
Here you have a function in Python 3, explain what it does
def restore(self, sess, save_path):

        if self._saver is None:
            raise TensorForceError("register_saver_ops should be called before restore")
        self._saver.restore(sess=sess, save_path=save_path)

Response: Restores the values of the managed variables from disk location.


Instruction:
Make a summary of the following Python 3 code
def reset(self):
        fetches = []
        for processor in self.preprocessors:
            fetches.extend(processor.reset() or [])
        return fetches

Response: Calls reset on all our preprocessors and returns a list of tensors to be fetched.


Instruction:
Make a summary of the following Python 3 code
def process(self, tensor):
        for processor in self.preprocessors:
            tensor = processor.process(tensor=tensor)
        return tensor

Response: Process a single state.


Instruction:
Write a Python 3 function that can
return the original shape of preprocessed state given original shape.

Response:
def processed_shape(self, shape):
        """
        Shape of preprocessed state given original shape.

        Args:
            shape: original state shape

        Returns: processed state shape
        """
        for processor in self.preprocessors:
            shape = processor.processed_shape(shape=shape)
        return shape


Instruction:
Explain what the following Python 3 code does
def from_spec(spec, kwargs=None):
        if isinstance(spec, dict):
            spec = [spec]

        stack = PreprocessorStack()
        for preprocessor_spec in spec:
            # need to deep copy, otherwise will add first processors spec_ to kwargs to second processor
            preprocessor_kwargs = copy.deepcopy(kwargs)
            preprocessor = util.get_object(
                obj=preprocessor_spec,
                predefined_objects=tensorforce.core.preprocessors.preprocessors,
                kwargs=preprocessor_kwargs
            )
            assert isinstance(preprocessor, Preprocessor)
            stack.preprocessors.append(preprocessor)

        return stack

Response: Creates a preprocessing stack from a specification dict.


Instruction:
Can you create a Python 3 function that
executes action and returns next state reward done

Response:
def execute(self, action):
        """
        Executes action, observes next state and reward.

        Args:
            actions: Actions to execute.

        Returns:
            Tuple of (next state, bool indicating terminal, reward)
        """
        next_state, rew, done, _ = self.env.step(action)
        return next_state, rew, done


Instruction:
Can you generate a brief explanation for the following Python 3 code
def as_local_model(self):
        super(MemoryModel, self).as_local_model()
        self.optimizer_spec = dict(
            type='global_optimizer',
            optimizer=self.optimizer_spec
        )

Response: Makes sure our optimizer is wrapped into the global_optimizer meta.


Instruction:
Create a Python 3 function to
construct the memory and optimizer objects and creates all template functions.

Response:
def setup_components_and_tf_funcs(self, custom_getter=None):
        """
        Constructs the memory and the optimizer objects.
        Generates and stores all template functions.
        """
        custom_getter = super(MemoryModel, self).setup_components_and_tf_funcs(custom_getter)

        # Memory
        self.memory = Memory.from_spec(
            spec=self.memory_spec,
            kwargs=dict(
                states=self.states_spec,
                internals=self.internals_spec,
                actions=self.actions_spec,
                summary_labels=self.summary_labels
            )
        )

        # Optimizer
        self.optimizer = Optimizer.from_spec(
            spec=self.optimizer_spec,
            kwargs=dict(summary_labels=self.summary_labels)
        )

        # TensorFlow functions
        self.fn_discounted_cumulative_reward = tf.make_template(
            name_='discounted-cumulative-reward',
            func_=self.tf_discounted_cumulative_reward,
            custom_getter_=custom_getter
        )
        self.fn_reference = tf.make_template(
            name_='reference',
            func_=self.tf_reference,
            custom_getter_=custom_getter
        )
        self.fn_loss_per_instance = tf.make_template(
            name_='loss-per-instance',
            func_=self.tf_loss_per_instance,
            custom_getter_=custom_getter
        )
        self.fn_regularization_losses = tf.make_template(
            name_='regularization-losses',
            func_=self.tf_regularization_losses,
            custom_getter_=custom_getter
        )
        self.fn_loss = tf.make_template(
            name_='loss',
            func_=self.tf_loss,
            custom_getter_=custom_getter
        )
        self.fn_optimization = tf.make_template(
            name_='optimization',
            func_=self.tf_optimization,
            custom_getter_=custom_getter
        )
        self.fn_import_experience = tf.make_template(
            name_='import-experience',
            func_=self.tf_import_experience,
            custom_getter_=custom_getter
        )

        return custom_getter


Instruction:
Explain what the following Python 3 code does
def tf_discounted_cumulative_reward(self, terminal, reward, discount=None, final_reward=0.0, horizon=0):

        # By default -> take Model's gamma value
        if discount is None:
            discount = self.discount

        # Accumulates discounted (n-step) reward (start new if terminal)
        def cumulate(cumulative, reward_terminal_horizon_subtract):
            rew, is_terminal, is_over_horizon, sub = reward_terminal_horizon_subtract
            return tf.where(
                # If terminal, start new cumulation.
                condition=is_terminal,
                x=rew,
                y=tf.where(
                    # If we are above the horizon length (H) -> subtract discounted value from H steps back.
                    condition=is_over_horizon,
                    x=(rew + cumulative * discount - sub),
                    y=(rew + cumulative * discount)
                )
            )

        # Accumulates length of episodes (starts new if terminal)
        def len_(cumulative, term):
            return tf.where(
                condition=term,
                # Start counting from 1 after is-terminal signal
                x=tf.ones(shape=(), dtype=tf.int32),
                # Otherwise, increase length by 1
                y=cumulative + 1
            )

        # Reverse, since reward cumulation is calculated right-to-left, but tf.scan only works left-to-right.
        reward = tf.reverse(tensor=reward, axis=(0,))
        # e.g. -1.0 1.0 0.5 0.0 1.0 2.0
        terminal = tf.reverse(tensor=terminal, axis=(0,))
        # e.g. F T F F F F

        # Store the steps until end of the episode(s) determined by the input terminal signals (True starts new count).
        lengths = tf.scan(fn=len_, elems=terminal, initializer=0)
        # e.g. 1 1 2 3 4 5
        off_horizon = tf.greater(lengths, tf.fill(dims=tf.shape(lengths), value=horizon))
        # e.g. F F F F T T

        # Calculate the horizon-subtraction value for each step.
        if horizon > 0:
            horizon_subtractions = tf.map_fn(lambda x: (discount ** horizon) * x, reward, dtype=tf.float32)
            # Shift right by size of horizon (fill rest with 0.0).
            horizon_subtractions = tf.concat([np.zeros(shape=(horizon,)), horizon_subtractions], axis=0)
            horizon_subtractions = tf.slice(horizon_subtractions, begin=(0,), size=tf.shape(reward))
            # e.g. 0.0, 0.0, 0.0, -1.0*g^3, 1.0*g^3, 0.5*g^3
        # all 0.0 if infinite horizon (special case: horizon=0)
        else:
            horizon_subtractions = tf.zeros(shape=tf.shape(reward))

        # Now do the scan, each time summing up the previous step (discounted by gamma) and
        # subtracting the respective `horizon_subtraction`.
        reward = tf.scan(
            fn=cumulate,
            elems=(reward, terminal, off_horizon, horizon_subtractions),
            initializer=final_reward if horizon != 1 else 0.0
        )
        # Re-reverse again to match input sequences.
        return tf.reverse(tensor=reward, axis=(0,))

Response: Creates and returns a TensorFlow operation for calculating the discounted cumulative reward for a given sequence of single rewards.


Instruction:
Write a Python 3 script for
returning a TensorFlow operation for obtaining the reference tensor for the given state internals actions terminal reward next_states next_internals update.

Response:
def tf_reference(self, states, internals, actions, terminal, reward, next_states, next_internals, update):
        """
        Creates the TensorFlow operations for obtaining the reference tensor(s), in case of a
        comparative loss.

        Args:
            states: Dict of state tensors.
            internals: List of prior internal state tensors.
            actions: Dict of action tensors.
            terminal: Terminal boolean tensor.
            reward: Reward tensor.
            next_states: Dict of successor state tensors.
            next_internals: List of posterior internal state tensors.
            update: Boolean tensor indicating whether this call happens during an update.

        Returns:
            Reference tensor(s).
        """
        return None


Instruction:
Create a Python 3 function to
return the TensorFlow operations for calculating the loss per instance.

Response:
def tf_loss_per_instance(self, states, internals, actions, terminal, reward,
                             next_states, next_internals, update, reference=None):
        """
        Creates the TensorFlow operations for calculating the loss per batch instance.

        Args:
            states: Dict of state tensors.
            internals: Dict of prior internal state tensors.
            actions: Dict of action tensors.
            terminal: Terminal boolean tensor.
            reward: Reward tensor.
            next_states: Dict of successor state tensors.
            next_internals: List of posterior internal state tensors.
            update: Boolean tensor indicating whether this call happens during an update.
            reference: Optional reference tensor(s), in case of a comparative loss.

        Returns:
            Loss per instance tensor.
        """
        raise NotImplementedError


Instruction:
Create a Python 3 function for
returning the TensorFlow loss for a single internal state.

Response:
def tf_loss(self, states, internals, actions, terminal, reward, next_states, next_internals, update, reference=None):
        """
        Creates the TensorFlow operations for calculating the full loss of a batch.

        Args:
            states: Dict of state tensors.
            internals: List of prior internal state tensors.
            actions: Dict of action tensors.
            terminal: Terminal boolean tensor.
            reward: Reward tensor.
            next_states: Dict of successor state tensors.
            next_internals: List of posterior internal state tensors.
            update: Boolean tensor indicating whether this call happens during an update.
            reference: Optional reference tensor(s), in case of a comparative loss.

        Returns:
            Loss tensor.
        """
        # Mean loss per instance
        loss_per_instance = self.fn_loss_per_instance(
            states=states,
            internals=internals,
            actions=actions,
            terminal=terminal,
            reward=reward,
            next_states=next_states,
            next_internals=next_internals,
            update=update,
            reference=reference
        )

        # Returns no-op.
        updated = self.memory.update_batch(loss_per_instance=loss_per_instance)
        with tf.control_dependencies(control_inputs=(updated,)):
            loss = tf.reduce_mean(input_tensor=loss_per_instance, axis=0)

            # Loss without regularization summary.
            if 'losses' in self.summary_labels:
                tf.contrib.summary.scalar(name='loss-without-regularization', tensor=loss)

            # Regularization losses.
            losses = self.fn_regularization_losses(states=states, internals=internals, update=update)
            if len(losses) > 0:
                loss += tf.add_n(inputs=[losses[name] for name in sorted(losses)])
                if 'regularization' in self.summary_labels:
                    for name in sorted(losses):
                        tf.contrib.summary.scalar(name=('regularization/' + name), tensor=losses[name])

            # Total loss summary.
            if 'losses' in self.summary_labels or 'total-loss' in self.summary_labels:
                tf.contrib.summary.scalar(name='total-loss', tensor=loss)

            return loss


Instruction:
Write a Python 3 script for
returning the arguments to be used as keyword arguments to the optimizer.

Response:
def optimizer_arguments(self, states, internals, actions, terminal, reward, next_states, next_internals):
        """
        Returns the optimizer arguments including the time, the list of variables to optimize,
        and various functions which the optimizer might require to perform an update step.

        Args:
            states (dict): Dict of state tensors.
            internals (dict): Dict of prior internal state tensors.
            actions (dict): Dict of action tensors.
            terminal: 1D boolean is-terminal tensor.
            reward: 1D (float) rewards tensor.
            next_states (dict): Dict of successor state tensors.
            next_internals (dict): Dict of posterior internal state tensors.

        Returns:
            Optimizer arguments as dict to be used as **kwargs to the optimizer.
        """
        arguments = dict(
            time=self.global_timestep,
            variables=self.get_variables(),
            arguments=dict(
                states=states,
                internals=internals,
                actions=actions,
                terminal=terminal,
                reward=reward,
                next_states=next_states,
                next_internals=next_internals,
                update=tf.constant(value=True)
            ),
            fn_reference=self.fn_reference,
            fn_loss=self.fn_loss
        )
        if self.global_model is not None:
            arguments['global_variables'] = self.global_model.get_variables()
        return arguments


Instruction:
Can you write a function in Python 3 where it
creates the TensorFlow operations for performing an optimization update step based on the given input states and actions batch. Args: states: Dict of state tensors. internals: List of prior internal state tensors. actions: Dict of action tensors. terminal: Terminal boolean tensor. reward: Reward tensor. next_states: Dict of successor state tensors. next_internals: List of posterior internal state tensors. Returns: The optimization operation.

Response:
def tf_optimization(self, states, internals, actions, terminal, reward, next_states=None, next_internals=None):
        """
        Creates the TensorFlow operations for performing an optimization update step based
        on the given input states and actions batch.

        Args:
            states: Dict of state tensors.
            internals: List of prior internal state tensors.
            actions: Dict of action tensors.
            terminal: Terminal boolean tensor.
            reward: Reward tensor.
            next_states: Dict of successor state tensors.
            next_internals: List of posterior internal state tensors.

        Returns:
            The optimization operation.
        """
        arguments = self.optimizer_arguments(
            states=states,
            internals=internals,
            actions=actions,
            terminal=terminal,
            reward=reward,
            next_states=next_states,
            next_internals=next_internals
        )
        return self.optimizer.minimize(**arguments)


Instruction:
Write a Python 3 function that can
create and returns the op that observes a timestep.

Response:
def tf_observe_timestep(self, states, internals, actions, terminal, reward):
        """
        Creates and returns the op that - if frequency condition is hit - pulls a batch from the memory
        and does one optimization step.
        """
        # Store timestep in memory
        stored = self.memory.store(
            states=states,
            internals=internals,
            actions=actions,
            terminal=terminal,
            reward=reward
        )

        # Periodic optimization
        with tf.control_dependencies(control_inputs=(stored,)):
            unit = self.update_mode['unit']
            batch_size = self.update_mode['batch_size']
            frequency = self.update_mode.get('frequency', batch_size)
            first_update = self.update_mode.get('first_update', 0)

            if unit == 'timesteps':
                # Timestep-based batch
                optimize = tf.logical_and(
                    x=tf.equal(x=(self.timestep % frequency), y=0),
                    y=tf.logical_and(
                        x=tf.greater_equal(x=self.timestep, y=batch_size),
                        y=tf.greater_equal(x=self.timestep, y=first_update)
                    )
                )

            elif unit == 'episodes':
                # Episode-based batch
                optimize = tf.logical_and(
                    x=tf.equal(x=(self.episode % frequency), y=0),
                    y=tf.logical_and(
                        # Only update once per episode increment.
                        x=tf.greater(x=tf.count_nonzero(input_tensor=terminal), y=0),
                        y=tf.logical_and(
                            x=tf.greater_equal(x=self.episode, y=batch_size),
                            y=tf.greater_equal(x=self.episode, y=first_update)
                        )
                    )
                )

            elif unit == 'sequences':
                # Timestep-sequence-based batch
                sequence_length = self.update_mode.get('length', 8)
                optimize = tf.logical_and(
                    x=tf.equal(x=(self.timestep % frequency), y=0),
                    y=tf.logical_and(
                        x=tf.greater_equal(x=self.timestep, y=(batch_size + sequence_length - 1)),
                        y=tf.greater_equal(x=self.timestep, y=first_update)
                    )
                )

            else:
                raise TensorForceError("Invalid update unit: {}.".format(unit))

            def true_fn():
                if unit == 'timesteps':
                    # Timestep-based batch
                    batch = self.memory.retrieve_timesteps(n=batch_size)
                elif unit == 'episodes':
                    # Episode-based batch
                    batch = self.memory.retrieve_episodes(n=batch_size)
                elif unit == 'sequences':
                    # Timestep-sequence-based batch
                    batch = self.memory.retrieve_sequences(n=batch_size, sequence_length=sequence_length)

                # Do not calculate gradients for memory-internal operations.
                batch = util.map_tensors(
                    fn=(lambda tensor: tf.stop_gradient(input=tensor)),
                    tensors=batch
                )

                optimize = self.fn_optimization(**batch)
                with tf.control_dependencies(control_inputs=(optimize,)):
                    return tf.logical_and(x=True, y=True)

            return tf.cond(pred=optimize, true_fn=true_fn, false_fn=tf.no_op)


Instruction:
Can you write a function in Python 3 where it
imports experiences into the TensorFlow memory structure.

Response:
def tf_import_experience(self, states, internals, actions, terminal, reward):
        """
        Imports experiences into the TensorFlow memory structure. Can be used to import
        off-policy data.

        :param states: Dict of state values to import with keys as state names and values as values to set.
        :param internals: Internal values to set, can be fetched from agent via agent.current_internals
            if no values available.
        :param actions: Dict of action values to import with keys as action names and values as values to set.
        :param terminal: Terminal value(s)
        :param reward: Reward value(s)
        """
        return self.memory.store(
            states=states,
            internals=internals,
            actions=actions,
            terminal=terminal,
            reward=reward
        )


Instruction:
Write a Python 3 script for
creating the TensorFlow operations for performing an optimization step. Args: time: Time tensor. variables: List of variables to optimize. arguments: Dict of arguments for callables, like fn_loss. fn_loss: A callable returning the loss of the current model. fn_reference: A callable returning the reference values, in case of a comparative loss. **kwargs: Additional arguments passed on to the internal optimizer. Returns: List of delta tensors corresponding to the updates for each optimized variable.

Response:
def tf_step(
        self,
        time,
        variables,
        arguments,
        fn_loss,
        fn_reference,
        **kwargs
    ):
        """
        Creates the TensorFlow operations for performing an optimization step.

        Args:
            time: Time tensor.
            variables: List of variables to optimize.
            arguments: Dict of arguments for callables, like fn_loss.
            fn_loss: A callable returning the loss of the current model.
            fn_reference: A callable returning the reference values, in case of a comparative loss.
            **kwargs: Additional arguments passed on to the internal optimizer.

        Returns:
            List of delta tensors corresponding to the updates for each optimized variable.
        """

        # Set reference to compare with at each optimization step, in case of a comparative loss.
        arguments['reference'] = fn_reference(**arguments)

        # Negative value since line search maximizes.
        loss_before = -fn_loss(**arguments)

        with tf.control_dependencies(control_inputs=(loss_before,)):
            deltas = self.optimizer.step(
                time=time,
                variables=variables,
                arguments=arguments,
                fn_loss=fn_loss,
                return_estimated_improvement=True,
                **kwargs
            )

            if isinstance(deltas, tuple):
                # If 'return_estimated_improvement' argument exists.
                if len(deltas) != 2:
                    raise TensorForceError("Unexpected output of internal optimizer.")
                deltas, estimated_improvement = deltas
                # Negative value since line search maximizes.
                estimated_improvement = -estimated_improvement
            else:
                estimated_improvement = None

        with tf.control_dependencies(control_inputs=deltas):
                # Negative value since line search maximizes.
            loss_step = -fn_loss(**arguments)

        with tf.control_dependencies(control_inputs=(loss_step,)):

            def evaluate_step(deltas):
                with tf.control_dependencies(control_inputs=deltas):
                    applied = self.apply_step(variables=variables, deltas=deltas)
                with tf.control_dependencies(control_inputs=(applied,)):
                    # Negative value since line search maximizes.
                    return -fn_loss(**arguments)

            return self.solver.solve(
                fn_x=evaluate_step,
                x_init=deltas,
                base_value=loss_before,
                target_value=loss_step,
                estimated_improvement=estimated_improvement
            )


Instruction:
Write a Python 3 script for
creating a Distribution object from a specification dict.

Response:
def from_spec(spec, kwargs=None):
        """
        Creates a distribution from a specification dict.
        """
        distribution = util.get_object(
            obj=spec,
            predefined_objects=tensorforce.core.distributions.distributions,
            kwargs=kwargs
        )
        assert isinstance(distribution, Distribution)
        return distribution


Instruction:
Create a Python 3 function for
resetting the agent s internal episode and time step counter and internal states.

Response:
def reset(self):
        """
        Resets the agent to its initial state (e.g. on experiment start). Updates the Model's internal episode and
        time step counter, internal states, and resets preprocessors.
        """
        self.episode, self.timestep, self.next_internals = self.model.reset()
        self.current_internals = self.next_internals


Instruction:
Make a summary of the following Python 3 code
def act(self, states, deterministic=False, independent=False, fetch_tensors=None, buffered=True, index=0):
        self.current_internals = self.next_internals

        if self.unique_state:
            self.current_states = dict(state=np.asarray(states))
        else:
            self.current_states = {name: np.asarray(states[name]) for name in sorted(states)}

        if fetch_tensors is not None:
            # Retrieve action
            self.current_actions, self.next_internals, self.timestep, self.fetched_tensors = self.model.act(
                states=self.current_states,
                internals=self.current_internals,
                deterministic=deterministic,
                independent=independent,
                fetch_tensors=fetch_tensors,
                index=index
            )

            if self.unique_action:
                return self.current_actions['action'], self.fetched_tensors
            else:
                return self.current_actions, self.fetched_tensors

        # Retrieve action.
        self.current_actions, self.next_internals, self.timestep = self.model.act(
            states=self.current_states,
            internals=self.current_internals,
            deterministic=deterministic,
            independent=independent,
            index=index
        )

        # Buffered mode only works single-threaded because buffer inserts
        # by multiple threads are non-atomic and can cause race conditions.
        if buffered:
            if self.unique_action:
                return self.current_actions['action']
            else:
                return self.current_actions
        else:
            if self.unique_action:
                return self.current_actions['action'], self.current_states, self.current_internals
            else:
                return self.current_actions, self.current_states, self.current_internals

Response: Returns the action of the given states.


Instruction:
Write a Python 3 function that can
observe an experience from the environment to learn from.

Response:
def observe(self, terminal, reward, index=0):
        """
        Observe experience from the environment to learn from. Optionally pre-processes rewards
        Child classes should call super to get the processed reward
        EX: terminal, reward = super()...

        Args:
            terminal (bool): boolean indicating if the episode terminated after the observation.
            reward (float): scalar reward that resulted from executing the action.
        """
        self.current_terminal = terminal
        self.current_reward = reward

        if self.batched_observe:
            # Batched observe for better performance with Python.
            self.observe_terminal[index].append(self.current_terminal)
            self.observe_reward[index].append(self.current_reward)

            if self.current_terminal or len(self.observe_terminal[index]) >= self.batching_capacity:
                self.episode = self.model.observe(
                    terminal=self.observe_terminal[index],
                    reward=self.observe_reward[index],
                    index=index
                )
                self.observe_terminal[index] = list()
                self.observe_reward[index] = list()

        else:
            self.episode = self.model.observe(
                terminal=self.current_terminal,
                reward=self.current_reward
            )


Instruction:
Can you tell what is the following Python 3 function doing
def atomic_observe(self, states, actions, internals, reward, terminal):
        # TODO probably unnecessary here.
        self.current_terminal = terminal
        self.current_reward = reward
        # print('action = {}'.format(actions))
        if self.unique_state:
            states = dict(state=states)
        if self.unique_action:
            actions = dict(action=actions)

        self.episode = self.model.atomic_observe(
            states=states,
            actions=actions,
            internals=internals,
            terminal=self.current_terminal,
            reward=self.current_reward
        )

Response: Utility method for unbuffered observing where each tuple is inserted into TensorFlow via a single session call.


Instruction:
Implement a Python 3 function for
saving the current state of the current model to a checkpoint file.

Response:
def save_model(self, directory=None, append_timestep=True):
        """
        Save TensorFlow model. If no checkpoint directory is given, the model's default saver
        directory is used. Optionally appends current timestep to prevent overwriting previous
        checkpoint files. Turn off to be able to load model from the same given path argument as
        given here.

        Args:
            directory (str): Optional checkpoint directory.
            append_timestep (bool):  Appends the current timestep to the checkpoint file if true.
                If this is set to True, the load path must include the checkpoint timestep suffix.
                For example, if stored to models/ and set to true, the exported file will be of the
                form models/model.ckpt-X where X is the last timestep saved. The load path must
                precisely match this file name. If this option is turned off, the checkpoint will
                always overwrite the file specified in path and the model can always be loaded under
                this path.

        Returns:
            Checkpoint path were the model was saved.
        """
        return self.model.save(directory=directory, append_timestep=append_timestep)


Instruction:
How would you explain what the following Python 3 function does
def restore_model(self, directory=None, file=None):
        self.model.restore(directory=directory, file=file)

Response:
Restore TensorFlow model. If no checkpoint file is given, the latest checkpoint is
        restored. If no checkpoint directory is given, the model's default saver directory is
        used (unless file specifies the entire path).

        Args:
            directory: Optional checkpoint directory.
            file: Optional checkpoint file, or path if directory not given.


Instruction:
Can you implement a function in Python 3 that
creates an agent from a specification dict.

Response:
def from_spec(spec, kwargs):
        """
        Creates an agent from a specification dict.
        """
        agent = util.get_object(
            obj=spec,
            predefined_objects=tensorforce.agents.agents,
            kwargs=kwargs
        )
        assert isinstance(agent, Agent)
        return agent


Instruction:
Here you have a function in Python 3, explain what it does
def get_named_tensor(self, name):
        if name in self.named_tensors:
            return True, self.named_tensors[name]
        else:
            return False, None

Response: Returns a named tensor if available.


Instruction:
Can you create a Python 3 function that
creates a network from a specification dict.

Response:
def from_spec(spec, kwargs=None):
        """
        Creates a network from a specification dict.
        """
        network = util.get_object(
            obj=spec,
            default_object=LayeredNetwork,
            kwargs=kwargs
        )
        assert isinstance(network, Network)
        return network


Instruction:
Implement a function in Python 3 to
store a transition in replay memory.

Response:
def put(self, item, priority=None):
        """
        Stores a transition in replay memory.

        If the memory is full, the oldest entry is replaced.
        """
        if not self._isfull():
            self._memory.append(None)
        position = self._next_position_then_increment()
        old_priority = 0 if self._memory[position] is None \
            else (self._memory[position].priority or 0)
        row = _SumRow(item, priority)
        self._memory[position] = row
        self._update_internal_nodes(
            position, (row.priority or 0) - old_priority)


Instruction:
Explain what the following Python 3 code does
def move(self, external_index, new_priority):
        index = external_index + (self._capacity - 1)
        return self._move(index, new_priority)

Response: Moves the priority of a leaf node.


Instruction:
Explain what the following Python 3 code does
def _move(self, index, new_priority):
        item, old_priority = self._memory[index]
        old_priority = old_priority or 0
        self._memory[index] = _SumRow(item, new_priority)
        self._update_internal_nodes(index, new_priority - old_priority)

Response: Move the priority of a leaf node.


Instruction:
Implement a Python 3 function for
updating internal nodes in the tree.

Response:
def _update_internal_nodes(self, index, delta):
        """
        Update internal priority sums when leaf priority has been changed.
        Args:
            index: leaf node index
            delta: change in priority
        """
        # Move up tree, increasing position, updating sum
        while index > 0:
            index = (index - 1) // 2
            self._memory[index] += delta


Instruction:
Make a summary of the following Python 3 code
def _next_position_then_increment(self):
        start = self._capacity - 1
        position = start + self._position
        self._position = (self._position + 1) % self._capacity
        return position

Response: Returns the next available position.


Instruction:
Make a summary of the following Python 3 code
def _sample_with_priority(self, p):
        parent = 0
        while True:
            left = 2 * parent + 1
            if left >= len(self._memory):
                # parent points to a leaf node already.
                return parent

            left_p = self._memory[left] if left < self._capacity - 1 \
                else (self._memory[left].priority or 0)
            if p <= left_p:
                parent = left
            else:
                if left + 1 >= len(self._memory):
                    raise RuntimeError('Right child is expected to exist.')
                p -= left_p
                parent = left + 1

Response: Sample random element with priority greater than p.


Instruction:
Write a Python 3 function for
sampling the minibatch of size batch_size.

Response:
def sample_minibatch(self, batch_size):
        """
        Sample minibatch of size batch_size.
        """
        pool_size = len(self)
        if pool_size == 0:
            return []

        delta_p = self._memory[0] / batch_size
        chosen_idx = []
        # if all priorities sum to ~0  choose randomly otherwise random sample
        if abs(self._memory[0]) < util.epsilon:
            chosen_idx = np.random.randint(self._capacity - 1, self._capacity - 1 + len(self), size=batch_size).tolist()
        else:
            for i in xrange(batch_size):
                lower = max(i * delta_p, 0)
                upper = min((i + 1) * delta_p, self._memory[0])
                p = random.uniform(lower, upper)
                chosen_idx.append(self._sample_with_priority(p))
        return [(i, self._memory[i]) for i in chosen_idx]


Instruction:
How would you code a function in Python 3 to
return a batch of the specified size according to priority.

Response:
def get_batch(self, batch_size, next_states=False):
        """
        Samples a batch of the specified size according to priority.

        Args:
            batch_size: The batch size
            next_states: A boolean flag indicating whether 'next_states' values should be included

        Returns: A dict containing states, actions, rewards, terminals, internal states (and next states)

        """
        if batch_size > len(self.observations):
            raise TensorForceError(
                "Requested batch size is larger than observations in memory: increase config.first_update.")

        # Init empty states
        states = {name: np.zeros((batch_size,) + tuple(state['shape']), dtype=util.np_dtype(
            state['type'])) for name, state in self.states_spec.items()}
        internals = [np.zeros((batch_size,) + shape, dtype)
                     for shape, dtype in self.internals_spec]
        actions = {name: np.zeros((batch_size,) + tuple(action['shape']), dtype=util.np_dtype(action['type'])) for name, action in self.actions_spec.items()}
        terminal = np.zeros((batch_size,), dtype=util.np_dtype('bool'))
        reward = np.zeros((batch_size,), dtype=util.np_dtype('float'))
        if next_states:
            next_states = {name: np.zeros((batch_size,) + tuple(state['shape']), dtype=util.np_dtype(
                state['type'])) for name, state in self.states_spec.items()}
            next_internals = [np.zeros((batch_size,) + shape, dtype)
                              for shape, dtype in self.internals_spec]

        # Start with unseen observations
        unseen_indices = list(xrange(
            self.none_priority_index + self.observations._capacity - 1,
            len(self.observations) + self.observations._capacity - 1)
        )
        self.batch_indices = unseen_indices[:batch_size]

        # Get remaining observations using weighted sampling
        remaining = batch_size - len(self.batch_indices)
        if remaining:
            samples = self.observations.sample_minibatch(remaining)
            sample_indices = [i for i, o in samples]
            self.batch_indices += sample_indices

        # Shuffle
        np.random.shuffle(self.batch_indices)

        # Collect observations
        for n, index in enumerate(self.batch_indices):
            observation, _ = self.observations._memory[index]

            for name, state in states.items():
                state[n] = observation[0][name]
            for k, internal in enumerate(internals):
                internal[n] = observation[1][k]
            for name, action in actions.items():
                action[n] = observation[2][name]
            terminal[n] = observation[3]
            reward[n] = observation[4]
            if next_states:
                for name, next_state in next_states.items():
                    next_state[n] = observation[5][name]
                for k, next_internal in enumerate(next_internals):
                    next_internal[n] = observation[6][k]

        if next_states:
            return dict(
                states=states,
                internals=internals,
                actions=actions,
                terminal=terminal,
                reward=reward,
                next_states=next_states,
                next_internals=next_internals
            )
        else:
            return dict(
                states=states,
                internals=internals,
                actions=actions,
                terminal=terminal,
                reward=reward
            )


Instruction:
Write a Python 3 function that can
update the internal state of the cluster with the given loss.

Response:
def update_batch(self, loss_per_instance):
        """
        Computes priorities according to loss.

        Args:
            loss_per_instance:

        """
        if self.batch_indices is None:
            raise TensorForceError("Need to call get_batch before each update_batch call.")
        # if len(loss_per_instance) != len(self.batch_indices):
        #     raise TensorForceError("For all instances a loss value has to be provided.")

        for index, loss in zip(self.batch_indices, loss_per_instance):
            # Sampling priority is proportional to the largest absolute temporal difference error.
            new_priority = (np.abs(loss) + self.prioritization_constant) ** self.prioritization_weight
            self.observations._move(index, new_priority)
            self.none_priority_index += 1


Instruction:
Write a Python 3 function that can
import the given list of experiences into the database.

Response:
def import_experience(self, experiences):
        """
        Imports experiences.

        Args:
            experiences:
        """
        if isinstance(experiences, dict):
            if self.unique_state:
                experiences['states'] = dict(state=experiences['states'])
            if self.unique_action:
                experiences['actions'] = dict(action=experiences['actions'])

            self.model.import_experience(**experiences)

        else:
            if self.unique_state:
                states = dict(state=list())
            else:
                states = {name: list() for name in experiences[0]['states']}
            internals = [list() for _ in experiences[0]['internals']]
            if self.unique_action:
                actions = dict(action=list())
            else:
                actions = {name: list() for name in experiences[0]['actions']}
            terminal = list()
            reward = list()

            for experience in experiences:
                if self.unique_state:
                    states['state'].append(experience['states'])
                else:
                    for name in sorted(states):
                        states[name].append(experience['states'][name])
                for n, internal in enumerate(internals):
                    internal.append(experience['internals'][n])
                if self.unique_action:
                    actions['action'].append(experience['actions'])
                else:
                    for name in sorted(actions):
                        actions[name].append(experience['actions'][name])
                terminal.append(experience['terminal'])
                reward.append(experience['reward'])

            self.model.import_experience(
                states=states,
                internals=internals,
                actions=actions,
                terminal=terminal,
                reward=reward
            )


Instruction:
How would you explain what the following Python 3 function does
def connect(self, timeout=600):
        # If we are already connected, return error.
        if self.socket:
            raise TensorForceError("Already connected to {}:{}. Only one connection allowed at a time. " +
                                   "Close first by calling `close`!".format(self.host, self.port))
        self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

        if timeout < 5 or timeout is None:
            timeout = 5

        err = 0
        start_time = time.time()
        while time.time() - start_time < timeout:
            self.socket.settimeout(5)
            err = self.socket.connect_ex((self.host, self.port))
            if err == 0:
                break
            time.sleep(1)
        if err != 0:
            raise TensorForceError("Error when trying to connect to {}:{}: errno={} errcode='{}' '{}'".
                                   format(self.host, self.port, err, errno.errorcode[err], os.strerror(err)))

Response: Connects to the remote server.


Instruction:
Implement a Python 3 function for
disconnecting from the server.

Response:
def disconnect(self):
        """
        Ends our server tcp connection.
        """
        # If we are not connected, return error.
        if not self.socket:
            logging.warning("No active socket to close!")
            return
        # Close our socket.
        self.socket.close()
        self.socket = None


Instruction:
Given the following Python 3 function, write the documentation
def send(self, message, socket_):
        if not socket_:
            raise TensorForceError("No socket given in call to `send`!")
        elif not isinstance(message, dict):
            raise TensorForceError("Message to be sent must be a dict!")
        message = msgpack.packb(message)
        len_ = len(message)
        # prepend 8-byte len field to all our messages
        socket_.send(bytes("{:08d}".format(len_), encoding="ascii") + message)

Response: Sends a message to the socket.


Instruction:
Write a Python 3 script for
receiving a message from the given socket and returns it as a dict.

Response:
def recv(self, socket_, encoding=None):
        """
        Receives a message as msgpack-numpy encoded byte-string from the given socket object.
        Blocks until something was received.

        Args:
            socket_: The python socket object to use.
            encoding (str): The encoding to use for unpacking messages from the socket.
        Returns: The decoded (as dict) message received.
        """
        unpacker = msgpack.Unpacker(encoding=encoding)

        # Wait for an immediate response.
        response = socket_.recv(8)  # get the length of the message
        if response == b"":
            raise TensorForceError("No data received by socket.recv in call to method `recv` " +
                                   "(listener possibly closed)!")
        orig_len = int(response)
        received_len = 0
        while True:
            data = socket_.recv(min(orig_len - received_len, self.max_msg_len))
            # There must be a response.
            if not data:
                raise TensorForceError("No data of len {} received by socket.recv in call to method `recv`!".
                                       format(orig_len - received_len))
            data_len = len(data)
            received_len += data_len
            unpacker.feed(data)

            if received_len == orig_len:
                break

        # Get the data.
        for message in unpacker:
            sts = message.get("status", message.get(b"status"))
            if sts:
                if sts == "ok" or sts == b"ok":
                    return message
                else:
                    raise TensorForceError("RemoteEnvironment server error: {}".
                                           format(message.get("message", "not specified")))
            else:
                raise TensorForceError("Message without field 'status' received!")
        raise TensorForceError("No message encoded in data stream (data stream had len={})".
                               format(orig_len))


Instruction:
Make a summary of the following Python 3 code
def is_action_available(self, action):

        temp_state = np.rot90(self._state, action)
        return self._is_action_available_left(temp_state)

Response: Determines whether the action is available.


Instruction:
Explain what the following Python 3 code does
def _is_action_available_left(self, state):

        # True if any field is 0 (empty) on the left of a tile or two tiles can
        # be merged.
        for row in range(4):
            has_empty = False
            for col in range(4):
                has_empty |= state[row, col] == 0
                if state[row, col] != 0 and has_empty:
                    return True
                if (state[row, col] != 0 and col > 0 and
                        state[row, col] == state[row, col - 1]):
                    return True

        return False

Response: Determines whether action Left is available.


Instruction:
Write a Python 3 function that can
execute action add a new tile update the score & return the reward.

Response:
def do_action(self, action):
        """Execute action, add a new tile, update the score & return the reward."""

        temp_state = np.rot90(self._state, action)
        reward = self._do_action_left(temp_state)
        self._state = np.rot90(temp_state, -action)
        self._score += reward

        self.add_random_tile()

        return reward


Instruction:
Write a Python 3 function for
adding a random tile to the grid. Assumes that the grid has empty fields.

Response:
def add_random_tile(self):
        """Adds a random tile to the grid. Assumes that it has empty fields."""

        x_pos, y_pos = np.where(self._state == 0)
        assert len(x_pos) != 0
        empty_index = np.random.choice(len(x_pos))
        value = np.random.choice([1, 2], p=[0.9, 0.1])

        self._state[x_pos[empty_index], y_pos[empty_index]] = value


Instruction:
Here you have a function in Python 3, explain what it does
def print_state(self):

        def tile_string(value):
            """Concert value to string."""
            if value > 0:
                return '% 5d' % (2 ** value,)
            return "     "

        separator_line = '-' * 25
        print(separator_line)
        for row in range(4):
            print("|" + "|".join([tile_string(v) for v in self._state[row, :]]) + "|")
            print(separator_line)

Response: Prints the current state of the current user.


Instruction:
Implement a function in Python 3 to
set up the TensorFlow model graph and creates the variables and summaries for the current model.

Response:
def setup(self):
        """
        Sets up the TensorFlow model graph, starts the servers (distributed mode), creates summarizers
        and savers, initializes (and enters) the TensorFlow session.
        """

        # Create/get our graph, setup local model/global model links, set scope and device.
        graph_default_context = self.setup_graph()

        # Start a tf Server (in case of distributed setup). Only start once.
        if self.execution_type == "distributed" and self.server is None and self.is_local_model:
            self.start_server()

        # build the graph
        with tf.device(device_name_or_function=self.device):
            with tf.variable_scope(name_or_scope=self.scope, reuse=False):

                # Variables and summaries
                self.variables = dict()
                self.all_variables = dict()
                self.registered_variables = set()

                # Build the graph's placeholders, tf_functions, etc
                self.setup_placeholders()
                # Create model's "external" components.
                # Create tensorflow functions from "tf_"-methods.
                self.setup_components_and_tf_funcs()

                # Create core variables (timestep, episode counters, buffers for states/actions/internals).
                self.fn_initialize()

                if self.summarizer_spec is not None:
                    with tf.name_scope(name='summarizer'):
                        self.summarizer = tf.contrib.summary.create_file_writer(
                            logdir=self.summarizer_spec['directory'],
                            max_queue=None,
                            flush_millis=(self.summarizer_spec.get('flush', 10) * 1000),
                            filename_suffix=None,
                            name=None
                        )
                        default_summarizer = self.summarizer.as_default()
                        # Problem: not all parts of the graph are called on every step
                        assert 'steps' not in self.summarizer_spec
                        # if 'steps' in self.summarizer_spec:
                        #     record_summaries = tf.contrib.summary.record_summaries_every_n_global_steps(
                        #         n=self.summarizer_spec['steps'],
                        #         global_step=self.global_timestep
                        #     )
                        # else:
                        record_summaries = tf.contrib.summary.always_record_summaries()

                    default_summarizer.__enter__()
                    record_summaries.__enter__()

                # Input tensors
                states = util.map_tensors(fn=tf.identity, tensors=self.states_input)
                internals = util.map_tensors(fn=tf.identity, tensors=self.internals_input)
                actions = util.map_tensors(fn=tf.identity, tensors=self.actions_input)
                terminal = tf.identity(input=self.terminal_input)
                reward = tf.identity(input=self.reward_input)
                # Probably both deterministic and independent should be the same at some point.
                deterministic = tf.identity(input=self.deterministic_input)
                independent = tf.identity(input=self.independent_input)
                episode_index = tf.identity(input=self.episode_index_input)

                states, actions, reward = self.fn_preprocess(states=states, actions=actions, reward=reward)

                self.create_operations(
                    states=states,
                    internals=internals,
                    actions=actions,
                    terminal=terminal,
                    reward=reward,
                    deterministic=deterministic,
                    independent=independent,
                    index=episode_index
                )

                # Add all summaries specified in summary_labels
                if 'inputs' in self.summary_labels or 'states' in self.summary_labels:
                    for name in sorted(states):
                        tf.contrib.summary.histogram(name=('states-' + name), tensor=states[name])
                if 'inputs' in self.summary_labels or 'actions' in self.summary_labels:
                    for name in sorted(actions):
                        tf.contrib.summary.histogram(name=('actions-' + name), tensor=actions[name])
                if 'inputs' in self.summary_labels or 'reward' in self.summary_labels:
                    tf.contrib.summary.histogram(name='reward', tensor=reward)

                if 'graph' in self.summary_labels:
                    with tf.name_scope(name='summarizer'):
                        graph_def = self.graph.as_graph_def()
                        graph_str = tf.constant(
                            value=graph_def.SerializeToString(),
                            dtype=tf.string,
                            shape=()
                        )
                        self.graph_summary = tf.contrib.summary.graph(
                            param=graph_str,
                            step=self.global_timestep
                        )
                        if 'meta_param_recorder_class' in self.summarizer_spec:
                            self.graph_summary = tf.group(
                                self.graph_summary,
                                *self.summarizer_spec['meta_param_recorder_class'].build_metagraph_list()
                            )

                if self.summarizer_spec is not None:
                    record_summaries.__exit__(None, None, None)
                    default_summarizer.__exit__(None, None, None)

                    with tf.name_scope(name='summarizer'):
                        self.flush_summarizer = tf.contrib.summary.flush()

                        self.summarizer_init_op = tf.contrib.summary.summary_writer_initializer_op()
                        assert len(self.summarizer_init_op) == 1
                        self.summarizer_init_op = self.summarizer_init_op[0]

        # If we are a global model -> return here.
        # Saving, syncing, finalizing graph, session is done by local replica model.
        if self.execution_type == "distributed" and not self.is_local_model:
            return

        # Saver/Summary -> Scaffold.
        self.setup_saver()

        self.setup_scaffold()

        # Create necessary hooks for the upcoming session.
        hooks = self.setup_hooks()

        # We are done constructing: Finalize our graph, create and enter the session.
        self.setup_session(self.server, hooks, graph_default_context)


Instruction:
How would you explain what the following Python 3 function does
def setup_graph(self):
        graph_default_context = None

        # Single (non-distributed) mode.
        if self.execution_type == "single":
            self.graph = tf.Graph()
            graph_default_context = self.graph.as_default()
            graph_default_context.__enter__()
            self.global_model = None

        # Distributed tf
        elif self.execution_type == "distributed":
            # Parameter-server -> Do not build any graph.
            if self.distributed_spec["job"] == "ps":
                return None

            # worker -> construct the global (main) model; the one hosted on the ps,
            elif self.distributed_spec["job"] == "worker":
                # The local replica model.
                if self.is_local_model:
                    graph = tf.Graph()
                    graph_default_context = graph.as_default()
                    graph_default_context.__enter__()

                    # Now that the graph is created and entered -> deepcopoy ourselves and setup global model first,
                    # then continue.
                    self.global_model = deepcopy(self)
                    # Switch on global construction/setup-mode for the pass to setup().
                    self.global_model.is_local_model = False
                    self.global_model.setup()

                    self.graph = graph
                    self.as_local_model()
                    self.scope += '-worker' + str(self.distributed_spec["task_index"])
                # The global_model (whose Variables are hosted by the ps).
                else:
                    self.graph = tf.get_default_graph()  # lives in the same graph as local model
                    self.global_model = None
                    self.device = tf.train.replica_device_setter(
                        # Place its Variables on the parameter server(s) (round robin).
                        #ps_device="/job:ps",  # default
                        # Train-ops for the global_model are hosted locally (on this worker's node).
                        worker_device=self.device,
                        cluster=self.distributed_spec["cluster_spec"]
                    )
            else:
                raise TensorForceError("Unsupported job type: {}!".format(self.distributed_spec["job"]))
        else:
            raise TensorForceError("Unsupported distributed type: {}!".format(self.distributed_spec["type"]))

        return graph_default_context

Response: Creates our Graph and figures out which global model to hook up to.


Instruction:
Given the following Python 3 function, write the documentation
def start_server(self):
        self.server = tf.train.Server(
            server_or_cluster_def=self.distributed_spec["cluster_spec"],
            job_name=self.distributed_spec["job"],
            task_index=self.distributed_spec["task_index"],
            protocol=self.distributed_spec.get("protocol"),
            config=self.distributed_spec.get("session_config"),
            start=True
        )
        if self.distributed_spec["job"] == "ps":
            self.server.join()
            # This is unreachable?
            quit()

Response: Creates and stores a tf. train. Server object and optionally joins it if we are a parameter - server.


Instruction:
Make a summary of the following Python 3 code
def setup_placeholders(self):

        # States
        for name in sorted(self.states_spec):
            self.states_input[name] = tf.placeholder(
                dtype=util.tf_dtype(self.states_spec[name]['type']),
                shape=(None,) + tuple(self.states_spec[name]['shape']),
                name=('state-' + name)
            )

        # States preprocessing
        if self.states_preprocessing_spec is None:
            for name in sorted(self.states_spec):
                self.states_spec[name]['unprocessed_shape'] = self.states_spec[name]['shape']
        elif not isinstance(self.states_preprocessing_spec, list) and \
                all(name in self.states_spec for name in self.states_preprocessing_spec):
            for name in sorted(self.states_spec):
                if name in self.states_preprocessing_spec:
                    preprocessing = PreprocessorStack.from_spec(
                        spec=self.states_preprocessing_spec[name],
                        kwargs=dict(shape=self.states_spec[name]['shape'])
                    )
                    self.states_spec[name]['unprocessed_shape'] = self.states_spec[name]['shape']
                    self.states_spec[name]['shape'] = preprocessing.processed_shape(shape=self.states_spec[name]['unprocessed_shape'])
                    self.states_preprocessing[name] = preprocessing
                else:
                    self.states_spec[name]['unprocessed_shape'] = self.states_spec[name]['shape']
        # Single preprocessor for all components of our state space
        elif "type" in self.states_preprocessing_spec:
            preprocessing = PreprocessorStack.from_spec(spec=self.states_preprocessing_spec,
                                                        kwargs=dict(shape=self.states_spec[name]['shape']))
            for name in sorted(self.states_spec):
                self.states_spec[name]['unprocessed_shape'] = self.states_spec[name]['shape']
                self.states_spec[name]['shape'] = preprocessing.processed_shape(shape=self.states_spec[name]['unprocessed_shape'])
                self.states_preprocessing[name] = preprocessing
        else:
            for name in sorted(self.states_spec):
                preprocessing = PreprocessorStack.from_spec(
                    spec=self.states_preprocessing_spec,
                    kwargs=dict(shape=self.states_spec[name]['shape'])
                )
                self.states_spec[name]['unprocessed_shape'] = self.states_spec[name]['shape']
                self.states_spec[name]['shape'] = preprocessing.processed_shape(shape=self.states_spec[name]['unprocessed_shape'])
                self.states_preprocessing[name] = preprocessing

        # Actions
        for name in sorted(self.actions_spec):
            self.actions_input[name] = tf.placeholder(
                dtype=util.tf_dtype(self.actions_spec[name]['type']),
                shape=(None,) + tuple(self.actions_spec[name]['shape']),
                name=('action-' + name)
            )

        # Actions exploration
        if self.actions_exploration_spec is None:
            pass
        elif all(name in self.actions_spec for name in self.actions_exploration_spec):
            for name in sorted(self.actions_spec):
                if name in self.actions_exploration:
                    self.actions_exploration[name] = Exploration.from_spec(spec=self.actions_exploration_spec[name])
        else:
            for name in sorted(self.actions_spec):
                self.actions_exploration[name] = Exploration.from_spec(spec=self.actions_exploration_spec)

        # Terminal
        self.terminal_input = tf.placeholder(dtype=util.tf_dtype('bool'), shape=(None,), name='terminal')

        # Reward
        self.reward_input = tf.placeholder(dtype=util.tf_dtype('float'), shape=(None,), name='reward')

        # Reward preprocessing
        if self.reward_preprocessing_spec is not None:
            self.reward_preprocessing = PreprocessorStack.from_spec(
                spec=self.reward_preprocessing_spec,
                # TODO this can eventually have more complex shapes?
                kwargs=dict(shape=())
            )
            if self.reward_preprocessing.processed_shape(shape=()) != ():
                raise TensorForceError("Invalid reward preprocessing!")

        # Deterministic/independent action flag (should probably be the same)
        self.deterministic_input = tf.placeholder(dtype=util.tf_dtype('bool'), shape=(), name='deterministic')
        self.independent_input = tf.placeholder(dtype=util.tf_dtype('bool'), shape=(), name='independent')

Response: Creates the TensorFlow placeholders variables ops and functions for the state space.


Instruction:
Write a Python 3 function that can
allow child models to create model's component objects, such as optimizer(s), memory(s), etc.. Creates all tensorflow functions via tf.make_template calls on all the class' "tf_"-methods. Args: custom_getter: The `custom_getter_` object to use for `tf.make_template` when creating TensorFlow functions. If None, use a default custom_getter_. Returns: The custom_getter passed in (or a default one if custom_getter was None).

Response:
def setup_components_and_tf_funcs(self, custom_getter=None):
        """
        Allows child models to create model's component objects, such as optimizer(s), memory(s), etc..
        Creates all tensorflow functions via tf.make_template calls on all the class' "tf_"-methods.

        Args:
            custom_getter: The `custom_getter_` object to use for `tf.make_template` when creating TensorFlow functions.
                If None, use a default custom_getter_.

        Returns: The custom_getter passed in (or a default one if custom_getter was None).
        """

        if custom_getter is None:
            def custom_getter(getter, name, registered=False, **kwargs):
                """
                To be passed to tf.make_template() as 'custom_getter_'.
                """
                if registered:
                    self.registered_variables.add(name)
                elif name in self.registered_variables:
                    registered = True
                # Top-level, hence no 'registered' argument.
                variable = getter(name=name, **kwargs)
                if registered:
                    pass
                elif name in self.all_variables:
                    assert variable is self.all_variables[name]
                    if kwargs.get('trainable', True):
                        assert variable is self.variables[name]
                        if 'variables' in self.summary_labels:
                            tf.contrib.summary.histogram(name=name, tensor=variable)
                else:
                    self.all_variables[name] = variable
                    if kwargs.get('trainable', True):
                        self.variables[name] = variable
                        if 'variables' in self.summary_labels:
                            tf.contrib.summary.histogram(name=name, tensor=variable)
                return variable

        self.fn_initialize = tf.make_template(
            name_='initialize',
            func_=self.tf_initialize,
            custom_getter_=custom_getter
        )
        self.fn_preprocess = tf.make_template(
            name_='preprocess',
            func_=self.tf_preprocess,
            custom_getter_=custom_getter
        )
        self.fn_actions_and_internals = tf.make_template(
            name_='actions-and-internals',
            func_=self.tf_actions_and_internals,
            custom_getter_=custom_getter
        )
        self.fn_observe_timestep = tf.make_template(
            name_='observe-timestep',
            func_=self.tf_observe_timestep,
            custom_getter_=custom_getter
        )
        self.fn_action_exploration = tf.make_template(
            name_='action-exploration',
            func_=self.tf_action_exploration,
            custom_getter_=custom_getter
        )

        return custom_getter


Instruction:
Make a summary of the following Python 3 code
def setup_saver(self):
        if self.execution_type == "single":
            global_variables = self.get_variables(include_submodules=True, include_nontrainable=True)
        else:
            global_variables = self.global_model.get_variables(include_submodules=True, include_nontrainable=True)

        # global_variables += [self.global_episode, self.global_timestep]

        for c in self.get_savable_components():
            c.register_saver_ops()

        # TensorFlow saver object
        # TODO potentially make other options configurable via saver spec.
        self.saver = tf.train.Saver(
            var_list=global_variables,  # should be given?
            reshape=False,
            sharded=False,
            max_to_keep=5,
            keep_checkpoint_every_n_hours=10000.0,
            name=None,
            restore_sequentially=False,
            saver_def=None,
            builder=None,
            defer_build=False,
            allow_empty=True,
            write_version=tf.train.SaverDef.V2,
            pad_step_number=False,
            save_relative_paths=True
            # filename=None
        )

Response: Creates the tf. train. Saver object and stores it in self. saver.


Instruction:
Can you create a Python 3 function that
creates the tf. train. Scaffold object and assigns it to self. scaffold.

Response:
def setup_scaffold(self):
        """
        Creates the tf.train.Scaffold object and assigns it to self.scaffold.
        Other fields of the Scaffold are generated automatically.
        """
        if self.execution_type == "single":
            global_variables = self.get_variables(include_submodules=True, include_nontrainable=True)
            # global_variables += [self.global_episode, self.global_timestep]
            init_op = tf.variables_initializer(var_list=global_variables)
            if self.summarizer_init_op is not None:
                init_op = tf.group(init_op, self.summarizer_init_op)
            if self.graph_summary is None:
                ready_op = tf.report_uninitialized_variables(var_list=global_variables)
                ready_for_local_init_op = None
                local_init_op = None
            else:
                ready_op = None
                ready_for_local_init_op = tf.report_uninitialized_variables(var_list=global_variables)
                local_init_op = self.graph_summary

        else:
            # Global and local variable initializers.
            global_variables = self.global_model.get_variables(include_submodules=True, include_nontrainable=True)
            # global_variables += [self.global_episode, self.global_timestep]
            local_variables = self.get_variables(include_submodules=True, include_nontrainable=True)
            init_op = tf.variables_initializer(var_list=global_variables)
            if self.summarizer_init_op is not None:
                init_op = tf.group(init_op, self.summarizer_init_op)
            ready_op = tf.report_uninitialized_variables(var_list=(global_variables + local_variables))
            ready_for_local_init_op = tf.report_uninitialized_variables(var_list=global_variables)
            if self.graph_summary is None:
                local_init_op = tf.group(
                    tf.variables_initializer(var_list=local_variables),
                    # Synchronize values of trainable variables.
                    *(tf.assign(ref=local_var, value=global_var) for local_var, global_var in zip(
                        self.get_variables(include_submodules=True),
                        self.global_model.get_variables(include_submodules=True)
                    ))
                )
            else:
                local_init_op = tf.group(
                    tf.variables_initializer(var_list=local_variables),
                    self.graph_summary,
                    # Synchronize values of trainable variables.
                    *(tf.assign(ref=local_var, value=global_var) for local_var, global_var in zip(
                        self.get_variables(include_submodules=True),
                        self.global_model.get_variables(include_submodules=True)
                    ))
                )

        def init_fn(scaffold, session):
            if self.saver_spec is not None and self.saver_spec.get('load', True):
                directory = self.saver_spec['directory']
                file = self.saver_spec.get('file')
                if file is None:
                    file = tf.train.latest_checkpoint(
                        checkpoint_dir=directory,
                        latest_filename=None  # Corresponds to argument of saver.save() in Model.save().
                    )
                elif not os.path.isfile(file):
                    file = os.path.join(directory, file)
                if file is not None:
                    try:
                        scaffold.saver.restore(sess=session, save_path=file)
                        session.run(fetches=self.list_buffer_index_reset_op)
                    except tf.errors.NotFoundError:
                        raise TensorForceError("Error: Existing checkpoint could not be loaded! Set \"load\" to false in saver_spec.")

        # TensorFlow scaffold object
        # TODO explain what it does.
        self.scaffold = tf.train.Scaffold(
            init_op=init_op,
            init_feed_dict=None,
            init_fn=init_fn,
            ready_op=ready_op,
            ready_for_local_init_op=ready_for_local_init_op,
            local_init_op=local_init_op,
            summary_op=None,
            saver=self.saver,
            copy_from_scaffold=None
        )


Instruction:
Can you tell what is the following Python 3 function doing
def setup_hooks(self):
        hooks = list()

        # Checkpoint saver hook
        if self.saver_spec is not None and (self.execution_type == 'single' or self.distributed_spec['task_index'] == 0):
            self.saver_directory = self.saver_spec['directory']
            hooks.append(tf.train.CheckpointSaverHook(
                checkpoint_dir=self.saver_directory,
                save_secs=self.saver_spec.get('seconds', None if 'steps' in self.saver_spec else 600),
                save_steps=self.saver_spec.get('steps'),  # Either one or the other has to be set.
                saver=None,  # None since given via 'scaffold' argument.
                checkpoint_basename=self.saver_spec.get('basename', 'model.ckpt'),
                scaffold=self.scaffold,
                listeners=None
            ))
        else:
            self.saver_directory = None

        # Stop at step hook
        # hooks.append(tf.train.StopAtStepHook(
        #     num_steps=???,  # This makes more sense, if load and continue training.
        #     last_step=None  # Either one or the other has to be set.
        # ))

        # # Step counter hook
        # hooks.append(tf.train.StepCounterHook(
        #     every_n_steps=counter_config.get('steps', 100),  # Either one or the other has to be set.
        #     every_n_secs=counter_config.get('secs'),  # Either one or the other has to be set.
        #     output_dir=None,  # None since given via 'summary_writer' argument.
        #     summary_writer=summary_writer
        # ))

        # Other available hooks:
        # tf.train.FinalOpsHook(final_ops, final_ops_feed_dict=None)
        # tf.train.GlobalStepWaiterHook(wait_until_step)
        # tf.train.LoggingTensorHook(tensors, every_n_iter=None, every_n_secs=None)
        # tf.train.NanTensorHook(loss_tensor, fail_on_nan_loss=True)
        # tf.train.ProfilerHook(save_steps=None, save_secs=None, output_dir='', show_dataflow=True, show_memory=False)

        return hooks

Response: Creates and returns a list of hooks to use in a session. Populates self. saver_directory.


Instruction:
How would you implement a function in Python 3 that
creates and enters the session for this model.

Response:
def setup_session(self, server, hooks, graph_default_context):
        """
        Creates and then enters the session for this model (finalizes the graph).

        Args:
            server (tf.train.Server): The tf.train.Server object to connect to (None for single execution).
            hooks (list): A list of (saver, summary, etc..) hooks to be passed to the session.
            graph_default_context: The graph as_default() context that we are currently in.
        """
        if self.execution_type == "distributed":
            # if self.distributed_spec['task_index'] == 0:
            # TensorFlow chief session creator object
            session_creator = tf.train.ChiefSessionCreator(
                scaffold=self.scaffold,
                master=server.target,
                config=self.session_config,
                checkpoint_dir=None,
                checkpoint_filename_with_path=None
            )
            # else:
            #     # TensorFlow worker session creator object
            #     session_creator = tf.train.WorkerSessionCreator(
            #         scaffold=self.scaffold,
            #         master=server.target,
            #         config=self.execution_spec.get('session_config'),
            #     )

            # TensorFlow monitored session object
            self.monitored_session = tf.train.MonitoredSession(
                session_creator=session_creator,
                hooks=hooks,
                stop_grace_period_secs=120  # Default value.
            )
            # Add debug session.run dumping?
            if self.tf_session_dump_dir != "":
                self.monitored_session = DumpingDebugWrapperSession(self.monitored_session, self.tf_session_dump_dir)
        else:
            # TensorFlow non-distributed monitored session object
            self.monitored_session = tf.train.SingularMonitoredSession(
                hooks=hooks,
                scaffold=self.scaffold,
                master='',  # Default value.
                config=self.session_config,  # self.execution_spec.get('session_config'),
                checkpoint_dir=None
            )

        if graph_default_context:
            graph_default_context.__exit__(None, None, None)
        self.graph.finalize()

        # enter the session to be ready for acting/learning
        self.monitored_session.__enter__()
        self.session = self.monitored_session._tf_sess()


Instruction:
How would you implement a function in Python 3 that
saves the model (of saver dir is given) and closes the session.

Response:
def close(self):
        """
        Saves the model (of saver dir is given) and closes the session.
        """
        if self.flush_summarizer is not None:
            self.monitored_session.run(fetches=self.flush_summarizer)
        if self.saver_directory is not None:
            self.save(append_timestep=True)
        self.monitored_session.__exit__(None, None, None)


Instruction:
How would you implement a function in Python 3 that
initializes the variables for the local state and internals and action buffers.

Response:
def tf_initialize(self):
        """
        Creates tf Variables for the local state/internals/action-buffers and for the local and global counters
        for timestep and episode.
        """

        # Timesteps/Episodes
        # Global: (force on global device; local and global model point to the same (global) data).
        with tf.device(device_name_or_function=(self.global_model.device if self.global_model else self.device)):

            # Global timestep
            collection = self.graph.get_collection(name='global-timestep')
            if len(collection) == 0:
                self.global_timestep = tf.get_variable(
                    name='global-timestep',
                    shape=(),
                    dtype=tf.int64,
                    trainable=False,
                    initializer=tf.constant_initializer(value=0, dtype=tf.int64),
                    collections=['global-timestep', tf.GraphKeys.GLOBAL_STEP]
                )
            else:
                assert len(collection) == 1
                self.global_timestep = collection[0]

            # Global episode
            collection = self.graph.get_collection(name='global-episode')
            if len(collection) == 0:
                self.global_episode = tf.get_variable(
                    name='global-episode',
                    shape=(),
                    dtype=tf.int64,
                    trainable=False,
                    initializer=tf.constant_initializer(value=0, dtype=tf.int64),
                    collections=['global-episode']
                )
            else:
                assert len(collection) == 1
                self.global_episode = collection[0]

        # Local counters: local device
        self.timestep = tf.get_variable(
            name='timestep',
            shape=(),
            dtype=tf.int64,
            initializer=tf.constant_initializer(value=0, dtype=tf.int64),
            trainable=False
        )

        self.episode = tf.get_variable(
            name='episode',
            shape=(),
            dtype=tf.int64,
            initializer=tf.constant_initializer(value=0, dtype=tf.int64),
            trainable=False
        )

        self.episode_index_input = tf.placeholder(
            name='episode_index',
            shape=(),
            dtype=tf.int32,
        )

        # States buffer variable
        for name in sorted(self.states_spec):
            self.list_states_buffer[name] = tf.get_variable(
                name=('state-{}'.format(name)),
                shape=((self.num_parallel, self.batching_capacity,) + tuple(self.states_spec[name]['shape'])),
                dtype=util.tf_dtype(self.states_spec[name]['type']),
                trainable=False
            )

        # Internals buffer variable
        for name in sorted(self.internals_spec):
            self.list_internals_buffer[name] = tf.get_variable(
                name=('internal-{}'.format(name)),
                shape=((self.num_parallel, self.batching_capacity,) + tuple(self.internals_spec[name]['shape'])),
                dtype=util.tf_dtype(self.internals_spec[name]['type']),
                trainable=False
            )

        # Actions buffer variable
        for name in sorted(self.actions_spec):
            self.list_actions_buffer[name]= tf.get_variable(
                name=('action-{}'.format(name)),
                shape=((self.num_parallel, self.batching_capacity,) + tuple(self.actions_spec[name]['shape'])),
                dtype=util.tf_dtype(self.actions_spec[name]['type']),
                trainable=False
            )

        # Buffer index
        # for index in range(self.num_parallel):
        self.list_buffer_index = tf.get_variable(
            name='buffer-index',
            shape=(self.num_parallel,),
            dtype=util.tf_dtype('int'),
            trainable=False
        )


Instruction:
Can you write a function in Python 3 where it
applies preprocessing ops to the raw states actions and reward tensors.

Response:
def tf_preprocess(self, states, actions, reward):
        """
        Applies preprocessing ops to the raw states/action/reward inputs.

        Args:
            states (dict): Dict of raw state tensors.
            actions (dict): Dict or raw action tensors.
            reward: 1D (float) raw rewards tensor.

        Returns: The preprocessed versions of the input tensors.
        """
        # States preprocessing
        for name in sorted(self.states_preprocessing):
            states[name] = self.states_preprocessing[name].process(tensor=states[name])

        # Reward preprocessing
        if self.reward_preprocessing is not None:
            reward = self.reward_preprocessing.process(tensor=reward)

        return states, actions, reward


Instruction:
Create a Python 3 function for
applying optional exploration to the action tensor.

Response:
def tf_action_exploration(self, action, exploration, action_spec):
        """
        Applies optional exploration to the action (post-processor for action outputs).

        Args:
             action (tf.Tensor): The original output action tensor (to be post-processed).
             exploration (Exploration): The Exploration object to use.
             action_spec (dict): Dict specifying the action space.
        Returns:
            The post-processed action output tensor.
        """
        action_shape = tf.shape(input=action)
        exploration_value = exploration.tf_explore(
            episode=self.global_episode,
            timestep=self.global_timestep,
            shape=action_spec['shape']
        )
        exploration_value = tf.expand_dims(input=exploration_value, axis=0)

        if action_spec['type'] == 'bool':
            action = tf.where(
                condition=(tf.random_uniform(shape=action_shape) < exploration_value),
                x=(tf.random_uniform(shape=action_shape) < 0.5),
                y=action
            )

        elif action_spec['type'] == 'int':
            action = tf.where(
                condition=(tf.random_uniform(shape=action_shape) < exploration_value),
                x=tf.random_uniform(shape=action_shape, maxval=action_spec['num_actions'], dtype=util.tf_dtype('int')),
                y=action
            )

        elif action_spec['type'] == 'float':
            noise = tf.random_normal(shape=action_shape, dtype=util.tf_dtype('float'))
            action += noise * exploration_value
            if 'min_value' in action_spec:
                action = tf.clip_by_value(
                    t=action,
                    clip_value_min=action_spec['min_value'],
                    clip_value_max=action_spec['max_value']
                )

        return action


Instruction:
Create a Python 3 function for
creating and stores tf operations that are fetched when calling act.

Response:
def create_act_operations(self, states, internals, deterministic, independent, index):
        """
        Creates and stores tf operations that are fetched when calling act(): actions_output, internals_output and
        timestep_output.

        Args:
            states (dict): Dict of state tensors (each key represents one state space component).
            internals (dict): Dict of prior internal state tensors (each key represents one internal state component).
            deterministic: 0D (bool) tensor (whether to not use action exploration).
            independent (bool): 0D (bool) tensor (whether to store states/internals/action in local buffer).
        """

        # Optional variable noise
        operations = list()
        if self.variable_noise is not None and self.variable_noise > 0.0:
            # Initialize variables
            self.fn_actions_and_internals(
                states=states,
                internals=internals,
                deterministic=deterministic
            )

            noise_deltas = list()
            for variable in self.get_variables():
                noise_delta = tf.random_normal(shape=util.shape(variable), mean=0.0, stddev=self.variable_noise)
                noise_deltas.append(noise_delta)
                operations.append(variable.assign_add(delta=noise_delta))

        # Retrieve actions and internals
        with tf.control_dependencies(control_inputs=operations):
            self.actions_output, self.internals_output = self.fn_actions_and_internals(
                states=states,
                internals=internals,
                deterministic=deterministic
            )

        # Subtract variable noise
        # TODO this is an untested/incomplete feature and maybe should be removed for now.
        with tf.control_dependencies(control_inputs=[self.actions_output[name] for name in sorted(self.actions_output)]):
            operations = list()
            if self.variable_noise is not None and self.variable_noise > 0.0:
                for variable, noise_delta in zip(self.get_variables(), noise_deltas):
                    operations.append(variable.assign_sub(delta=noise_delta))

        # Actions exploration
        with tf.control_dependencies(control_inputs=operations):
            for name in sorted(self.actions_exploration):
                self.actions_output[name] = tf.cond(
                    pred=self.deterministic_input,
                    true_fn=(lambda: self.actions_output[name]),
                    false_fn=(lambda: self.fn_action_exploration(
                        action=self.actions_output[name],
                        exploration=self.actions_exploration[name],
                        action_spec=self.actions_spec[name]
                    ))
                )

        # Independent act not followed by observe.
        def independent_act():
            """
            Does not store state, action, internal in buffer. Hence, does not have any influence on learning.
            Does not increase timesteps.
            """
            return self.global_timestep

        # Normal act followed by observe, with additional operations.
        def normal_act():
            """
            Stores current states, internals and actions in buffer. Increases timesteps.
            """
            operations = list()

            batch_size = tf.shape(input=states[next(iter(sorted(states)))])[0]
            for name in sorted(states):
                operations.append(tf.assign(
                    ref=self.list_states_buffer[name][index, self.list_buffer_index[index]: self.list_buffer_index[index] + batch_size],
                    value=states[name]
                ))
            for name in sorted(internals):
                operations.append(tf.assign(
                    ref=self.list_internals_buffer[name][index, self.list_buffer_index[index]: self.list_buffer_index[index] + batch_size],
                    value=internals[name]
                ))
            for name in sorted(self.actions_output):
                operations.append(tf.assign(
                    ref=self.list_actions_buffer[name][index, self.list_buffer_index[index]: self.list_buffer_index[index] + batch_size],
                    value=self.actions_output[name]
                ))

            with tf.control_dependencies(control_inputs=operations):
                operations = list()

                operations.append(tf.assign(
                    ref=self.list_buffer_index[index: index+1],
                    value=tf.add(self.list_buffer_index[index: index+1], tf.constant([1]))
                ))

                    # Increment timestep
                operations.append(tf.assign_add(
                    ref=self.timestep,
                    value=tf.to_int64(x=batch_size)
                ))
                operations.append(tf.assign_add(
                    ref=self.global_timestep,
                    value=tf.to_int64(x=batch_size)
                ))

            with tf.control_dependencies(control_inputs=operations):
                # Trivial operation to enforce control dependency
                # TODO why not return no-op?
                return self.global_timestep + 0

        # Only increment timestep and update buffer if act not independent
        self.timestep_output = tf.cond(
            pred=independent,
            true_fn=independent_act,
            false_fn=normal_act
        )