Instruction:
Implement a function in Python 3 to
create the operations needed to run an observation batch.

Response:
def create_observe_operations(self, terminal, reward, index):
        """
        Returns the tf op to fetch when an observation batch is passed in (e.g. an episode's rewards and
        terminals). Uses the filled tf buffers for states, actions and internals to run
        the tf_observe_timestep (model-dependent), resets buffer index and increases counters (episodes,
        timesteps).

        Args:
            terminal: The 1D tensor (bool) of terminal signals to process (more than one True within that list is ok).
            reward: The 1D tensor (float) of rewards to process.

        Returns: Tf op to fetch when `observe()` is called.
        """
        # Increment episode
        num_episodes = tf.count_nonzero(input_tensor=terminal, dtype=util.tf_dtype('int'))
        increment_episode = tf.assign_add(ref=self.episode, value=tf.to_int64(x=num_episodes))
        increment_global_episode = tf.assign_add(ref=self.global_episode, value=tf.to_int64(x=num_episodes))

        with tf.control_dependencies(control_inputs=(increment_episode, increment_global_episode)):
            # Stop gradients
            fn = (lambda x: tf.stop_gradient(input=x[:self.list_buffer_index[index]]))
            states = util.map_tensors(fn=fn, tensors=self.list_states_buffer, index=index)
            internals = util.map_tensors(fn=fn, tensors=self.list_internals_buffer, index=index)
            actions = util.map_tensors(fn=fn, tensors=self.list_actions_buffer, index=index)
            terminal = tf.stop_gradient(input=terminal)
            reward = tf.stop_gradient(input=reward)

            # Observation
            observation = self.fn_observe_timestep(
                states=states,
                internals=internals,
                actions=actions,
                terminal=terminal,
                reward=reward
            )

        with tf.control_dependencies(control_inputs=(observation,)):
            # Reset buffer index.
            reset_index = tf.assign(ref=self.list_buffer_index[index], value=0)

        with tf.control_dependencies(control_inputs=(reset_index,)):
            # Trivial operation to enforce control dependency.
            self.episode_output = self.global_episode + 0

        self.list_buffer_index_reset_op = tf.group(
            *(tf.assign(ref=self.list_buffer_index[n], value=0) for n in range(self.num_parallel))
        )


Instruction:
Can you generate a brief explanation for the following Python 3 code
def create_atomic_observe_operations(self, states, actions, internals, terminal, reward, index):
        # Increment episode
        num_episodes = tf.count_nonzero(input_tensor=terminal, dtype=util.tf_dtype('int'))
        increment_episode = tf.assign_add(ref=self.episode, value=tf.to_int64(x=num_episodes))
        increment_global_episode = tf.assign_add(ref=self.global_episode, value=tf.to_int64(x=num_episodes))

        with tf.control_dependencies(control_inputs=(increment_episode, increment_global_episode)):
            # Stop gradients
            # Not using buffers here.
            states = util.map_tensors(fn=tf.stop_gradient, tensors=states)
            internals = util.map_tensors(fn=tf.stop_gradient, tensors=internals)
            actions = util.map_tensors(fn=tf.stop_gradient, tensors=actions)
            terminal = tf.stop_gradient(input=terminal)
            reward = tf.stop_gradient(input=reward)

            # Observation
            observation = self.fn_observe_timestep(
                states=states,
                internals=internals,
                actions=actions,
                terminal=terminal,
                reward=reward
            )

        with tf.control_dependencies(control_inputs=(observation,)):
            # Trivial operation to enforce control dependency.
            self.unbuffered_episode_output = self.global_episode + 0

Response: Creates the atomic operations needed to fetch unbuffered observations.


Instruction:
Given the following Python 3 function, write the documentation
def create_operations(self, states, internals, actions, terminal, reward, deterministic, independent, index):
        self.create_act_operations(
            states=states,
            internals=internals,
            deterministic=deterministic,
            independent=independent,
            index=index
        )
        self.create_observe_operations(
            reward=reward,
            terminal=terminal,
            index=index
        )
        self.create_atomic_observe_operations(
            states=states,
            actions=actions,
            internals=internals,
            reward=reward,
            terminal=terminal,
            index=index
        )

Response: Creates and stores tf operations for when act and observe are called.


Instruction:
How would you code a function in Python 3 to
return the TensorFlow variables used by the model.

Response:
def get_variables(self, include_submodules=False, include_nontrainable=False):
        """
        Returns the TensorFlow variables used by the model.

        Args:
            include_submodules: Includes variables of submodules (e.g. baseline, target network)
                if true.
            include_nontrainable: Includes non-trainable variables if true.

        Returns:
            List of variables.
        """
        if include_nontrainable:
            model_variables = [self.all_variables[key] for key in sorted(self.all_variables)]

            states_preprocessing_variables = [
                variable for name in sorted(self.states_preprocessing)
                for variable in self.states_preprocessing[name].get_variables()
            ]
            model_variables += states_preprocessing_variables

            actions_exploration_variables = [
                variable for name in sorted(self.actions_exploration)
                for variable in self.actions_exploration[name].get_variables()
            ]
            model_variables += actions_exploration_variables

            if self.reward_preprocessing is not None:
                reward_preprocessing_variables = self.reward_preprocessing.get_variables()
                model_variables += reward_preprocessing_variables

        else:
            model_variables = [self.variables[key] for key in sorted(self.variables)]

        return model_variables


Instruction:
Can you tell what is the following Python 3 function doing
def reset(self):
        fetches = [self.global_episode, self.global_timestep]

        # Loop through all preprocessors and reset them as well.
        for name in sorted(self.states_preprocessing):
            fetch = self.states_preprocessing[name].reset()
            if fetch is not None:
                fetches.extend(fetch)

        if self.flush_summarizer is not None:
            fetches.append(self.flush_summarizer)

        # Get the updated episode and timestep counts.
        fetch_list = self.monitored_session.run(fetches=fetches)
        episode, timestep = fetch_list[:2]

        return episode, timestep, self.internals_init

Response: Resets the model to its initial state on episode start. This should also reset all preprocessor and state.


Instruction:
Write a Python 3 script to
return the feed - dict for the model s acting and observing tf fetches.

Response:
def get_feed_dict(
        self,
        states=None,
        internals=None,
        actions=None,
        terminal=None,
        reward=None,
        deterministic=None,
        independent=None,
        index=None
    ):
        """
        Returns the feed-dict for the model's acting and observing tf fetches.

        Args:
            states (dict): Dict of state values (each key represents one state space component).
            internals (dict): Dict of internal state values (each key represents one internal state component).
            actions (dict): Dict of actions (each key represents one action space component).
            terminal (List[bool]): List of is-terminal signals.
            reward (List[float]): List of reward signals.
            deterministic (bool): Whether actions should be picked without exploration.
            independent (bool): Whether we are doing an independent act (not followed by call to observe;
                not to be stored in model's buffer).

        Returns: The feed dict to use for the fetch.
        """
        feed_dict = dict()
        batched = None

        if states is not None:
            if batched is None:
                name = next(iter(states))
                state = np.asarray(states[name])
                batched = (state.ndim != len(self.states_spec[name]['unprocessed_shape']))
            if batched:
                feed_dict.update({self.states_input[name]: states[name] for name in sorted(self.states_input)})
            else:
                feed_dict.update({self.states_input[name]: (states[name],) for name in sorted(self.states_input)})

        if internals is not None:
            if batched is None:
                name = next(iter(internals))
                internal = np.asarray(internals[name])
                batched = (internal.ndim != len(self.internals_spec[name]['shape']))
            if batched:
                feed_dict.update({self.internals_input[name]: internals[name] for name in sorted(self.internals_input)})
            else:
                feed_dict.update({self.internals_input[name]: (internals[name],) for name in sorted(self.internals_input)})

        if actions is not None:
            if batched is None:
                name = next(iter(actions))
                action = np.asarray(actions[name])
                batched = (action.ndim != len(self.actions_spec[name]['shape']))
            if batched:
                feed_dict.update({self.actions_input[name]: actions[name] for name in sorted(self.actions_input)})
            else:
                feed_dict.update({self.actions_input[name]: (actions[name],) for name in sorted(self.actions_input)})

        if terminal is not None:
            if batched is None:
                terminal = np.asarray(terminal)
                batched = (terminal.ndim == 1)
            if batched:
                feed_dict[self.terminal_input] = terminal
            else:
                feed_dict[self.terminal_input] = (terminal,)

        if reward is not None:
            if batched is None:
                reward = np.asarray(reward)
                batched = (reward.ndim == 1)
            if batched:
                feed_dict[self.reward_input] = reward
            else:
                feed_dict[self.reward_input] = (reward,)

        if deterministic is not None:
            feed_dict[self.deterministic_input] = deterministic

        if independent is not None:
            feed_dict[self.independent_input] = independent

        feed_dict[self.episode_index_input] = index

        return feed_dict


Instruction:
Implement a function in Python 3 to
do a forward pass through the model to retrieve action (outputs) given inputs for state (and internal state, if applicable (e.g. RNNs)) Args: states (dict): Dict of state values (each key represents one state space component). internals (dict): Dict of internal state values (each key represents one internal state component). deterministic (bool): If True, will not apply exploration after actions are calculated. independent (bool): If true, action is not followed by observe (and hence not included in updates). fetch_tensors (list): List of names of additional tensors (from the model's network) to fetch (and return). index: (int) index of the episode we want to produce the next action Returns: tuple: - Actual action-outputs (batched if state input is a batch). - Actual values of internal states (if applicable) (batched if state input is a batch). - The timestep (int) after calculating the (batch of) action(s).

Response:
def act(self, states, internals, deterministic=False, independent=False, fetch_tensors=None, index=0):
        """
        Does a forward pass through the model to retrieve action (outputs) given inputs for state (and internal
        state, if applicable (e.g. RNNs))

        Args:
            states (dict): Dict of state values (each key represents one state space component).
            internals (dict): Dict of internal state values (each key represents one internal state component).
            deterministic (bool): If True, will not apply exploration after actions are calculated.
            independent (bool): If true, action is not followed by observe (and hence not included
                in updates).
            fetch_tensors (list): List of names of additional tensors (from the model's network) to fetch (and return).
            index: (int) index of the episode we want to produce the next action

        Returns:
            tuple:
                - Actual action-outputs (batched if state input is a batch).
                - Actual values of internal states (if applicable) (batched if state input is a batch).
                - The timestep (int) after calculating the (batch of) action(s).
        """
        name = next(iter(states))
        state = np.asarray(states[name])
        batched = (state.ndim != len(self.states_spec[name]['unprocessed_shape']))
        if batched:
            assert state.shape[0] <= self.batching_capacity

        fetches = [self.actions_output, self.internals_output, self.timestep_output]
        if self.network is not None and fetch_tensors is not None:
            for name in fetch_tensors:
                valid, tensor = self.network.get_named_tensor(name)
                if valid:
                    fetches.append(tensor)
                else:
                    keys = self.network.get_list_of_named_tensor()
                    raise TensorForceError('Cannot fetch named tensor "{}", Available {}.'.format(name, keys))

        # feed_dict[self.deterministic_input] = deterministic
        feed_dict = self.get_feed_dict(
            states=states,
            internals=internals,
            deterministic=deterministic,
            independent=independent,
            index=index
        )

        fetch_list = self.monitored_session.run(fetches=fetches, feed_dict=feed_dict)
        actions, internals, timestep = fetch_list[0:3]

        # Extract the first (and only) action/internal from the batch to make return values non-batched
        if not batched:
            actions = {name: actions[name][0] for name in sorted(actions)}
            internals = {name: internals[name][0] for name in sorted(internals)}

        if self.network is not None and fetch_tensors is not None:
            fetch_dict = dict()
            for index_, tensor in enumerate(fetch_list[3:]):
                name = fetch_tensors[index_]
                fetch_dict[name] = tensor
            return actions, internals, timestep, fetch_dict
        else:
            return actions, internals, timestep


Instruction:
Write a Python 3 script for
adding an observation to the model without updating its trainable variables.

Response:
def observe(self, terminal, reward, index=0):
        """
        Adds an observation (reward and is-terminal) to the model without updating its trainable variables.

        Args:
            terminal (List[bool]): List of is-terminal signals.
            reward (List[float]): List of reward signals.
            index: (int) parallel episode you want to observe

        Returns:
            The value of the model-internal episode counter.
        """
        fetches = self.episode_output
        feed_dict = self.get_feed_dict(terminal=terminal, reward=reward, index=index)

        episode = self.monitored_session.run(fetches=fetches, feed_dict=feed_dict)

        return episode


Instruction:
How would you implement a function in Python 3 that
saves the current state of the current model to a checkpoint file.

Response:
def save(self, directory=None, append_timestep=True):
        """
        Save TensorFlow model. If no checkpoint directory is given, the model's default saver
        directory is used. Optionally appends current timestep to prevent overwriting previous
        checkpoint files. Turn off to be able to load model from the same given path argument as
        given here.

        Args:
            directory: Optional checkpoint directory.
            append_timestep: Appends the current timestep to the checkpoint file if true.

        Returns:
            Checkpoint path where the model was saved.
        """
        if self.flush_summarizer is not None:
            self.monitored_session.run(fetches=self.flush_summarizer)

        return self.saver.save(
            sess=self.session,
            save_path=(self.saver_directory if directory is None else directory),
            global_step=(self.global_timestep if append_timestep else None),
            # latest_filename=None,  # Defaults to 'checkpoint'.
            meta_graph_suffix='meta',
            write_meta_graph=True,
            write_state=True
        )


Instruction:
Can you generate a brief explanation for the following Python 3 code
def restore(self, directory=None, file=None):
        if file is None:
            file = tf.train.latest_checkpoint(
                checkpoint_dir=(self.saver_directory if directory is None else directory),
                # latest_filename=None  # Corresponds to argument of saver.save() in Model.save().
            )
        elif directory is None:
            file = os.path.join(self.saver_directory, file)
        elif not os.path.isfile(file):
            file = os.path.join(directory, file)

        # if not os.path.isfile(file):
        #     raise TensorForceError("Invalid model directory/file.")

        self.saver.restore(sess=self.session, save_path=file)
        self.session.run(fetches=self.list_buffer_index_reset_op)

Response:
Restore TensorFlow model. If no checkpoint file is given, the latest checkpoint is
        restored. If no checkpoint directory is given, the model's default saver directory is
        used (unless file specifies the entire path).

        Args:
            directory: Optional checkpoint directory.
            file: Optional checkpoint file, or path if directory not given.


Instruction:
Create a Python 3 function for
returning the list of all of the components that can be individually saved and restored.

Response:
def get_savable_components(self):
        """
        Returns the list of all of the components this model consists of that can be individually saved and restored.
        For instance the network or distribution.

        Returns:
            List of util.SavableComponent
        """
        components = self.get_components()
        components = [components[name] for name in sorted(components)]
        return set(filter(lambda x: isinstance(x, util.SavableComponent), components))


Instruction:
Explain what the following Python 3 code does
def save_component(self, component_name, save_path):
        component = self.get_component(component_name=component_name)
        self._validate_savable(component=component, component_name=component_name)
        return component.save(sess=self.session, save_path=save_path)

Response: Saves a component of this model to the designated location.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def restore_component(self, component_name, save_path):
        component = self.get_component(component_name=component_name)
        self._validate_savable(component=component, component_name=component_name)
        component.restore(sess=self.session, save_path=save_path)

Response: Restores a component s parameters from a save location.


Instruction:
Write a Python 3 function that can
return the component with the provided name or None if there is no such component.

Response:
def get_component(self, component_name):
        """
        Looks up a component by its name.

        Args:
            component_name: The name of the component to look up.
        Returns:
            The component for the provided name or None if there is no such component.
        """
        mapping = self.get_components()
        return mapping[component_name] if component_name in mapping else None


Instruction:
How would you implement a function in Python 3 that
imports the demonstrations in the database into memory.

Response:
def import_demonstrations(self, demonstrations):
        """
        Imports demonstrations, i.e. expert observations. Note that for large numbers of observations,
        set_demonstrations is more appropriate, which directly sets memory contents to an array an expects
        a different layout.

        Args:
            demonstrations: List of observation dicts
        """
        if isinstance(demonstrations, dict):
            if self.unique_state:
                demonstrations['states'] = dict(state=demonstrations['states'])
            if self.unique_action:
                demonstrations['actions'] = dict(action=demonstrations['actions'])

            self.model.import_demo_experience(**demonstrations)

        else:
            if self.unique_state:
                states = dict(state=list())
            else:
                states = {name: list() for name in demonstrations[0]['states']}
            internals = {name: list() for name in demonstrations[0]['internals']}
            if self.unique_action:
                actions = dict(action=list())
            else:
                actions = {name: list() for name in demonstrations[0]['actions']}
            terminal = list()
            reward = list()

            for demonstration in demonstrations:
                if self.unique_state:
                    states['state'].append(demonstration['states'])
                else:
                    for name, state in states.items():
                        state.append(demonstration['states'][name])
                for name, internal in internals.items():
                    internal.append(demonstration['internals'][name])
                if self.unique_action:
                    actions['action'].append(demonstration['actions'])
                else:
                    for name, action in actions.items():
                        action.append(demonstration['actions'][name])
                terminal.append(demonstration['terminal'])
                reward.append(demonstration['reward'])

            self.model.import_demo_experience(
                states=states,
                internals=internals,
                actions=actions,
                terminal=terminal,
                reward=reward
            )


Instruction:
Explain what the following Python 3 code does
def seed(self, seed): # pylint: disable=E0202
        if seed is None:
            self.env.seed = round(time.time())
        else:
            self.env.seed = seed
        return self.env.seed

Response: Sets the random seed of the environment.


Instruction:
Write a Python 3 script for
executing action and returns next state done and reward.

Response:
def execute(self, action):
        """
        Executes action, observes next state and reward.

        Args:
            actions: Action to execute.

        Returns:
            (Dict of) next state(s), boolean indicating terminal, and reward signal.
        """
        if self.env.game_over():
            return self.env.getScreenRGB(), True, 0

        action_space = self.env.getActionSet()
        reward = self.env.act(action_space[action])
        new_state = self.env.getScreenRGB()
        done = self.env.game_over()
        return new_state, done, reward


Instruction:
How would you implement a function in Python 3 that
returns the state space. Might include subdicts.

Response:
def states(self):
        """
        Return the state space. Might include subdicts if multiple states are
        available simultaneously.

        Returns: dict of state properties (shape and type).

        """
        screen = self.env.getScreenRGB()
        return dict(shape=screen.shape, type='int')


Instruction:
Can you generate the documentation for the following Python 3 function
def sanity_check_actions(actions_spec):
    # Leave incoming spec-dict intact.
    actions = copy.deepcopy(actions_spec)

    # Unique action shortform.
    is_unique = ('type' in actions)
    if is_unique:
        actions = dict(action=actions)

    # Normalize actions.
    for name, action in actions.items():
        # Set default type to int
        if 'type' not in action:
            action['type'] = 'int'

        # Check required values
        if action['type'] == 'int':
            if 'num_actions' not in action:
                raise TensorForceError("Action requires value 'num_actions' set!")
        elif action['type'] == 'float':
            if ('min_value' in action) != ('max_value' in action):
                raise TensorForceError("Action requires both values 'min_value' and 'max_value' set!")

        # Set default shape to empty tuple (single-int, discrete action space)
        if 'shape' not in action:
            action['shape'] = ()

        # Convert int to unary tuple
        if isinstance(action['shape'], int):
            action['shape'] = (action['shape'],)

    return actions, is_unique

Response: Sanity checks an actions dict for MDP.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def sanity_check_execution_spec(execution_spec):

    # default spec: single mode
    def_ = dict(type="single",
                distributed_spec=None,
                session_config=None)

    if execution_spec is None:
        return def_

    assert isinstance(execution_spec, dict), "ERROR: execution-spec needs to be of type dict (but is of type {})!".\
        format(type(execution_spec).__name__)

    type_ = execution_spec.get("type")

    # TODO: Figure out what exactly we need for options and what types we should support.
    if type_ == "distributed":
        def_ = dict(job="ps", task_index=0, cluster_spec={
            "ps": ["localhost:22222"],
            "worker": ["localhost:22223"]
        })
        def_.update(execution_spec.get("distributed_spec", {}))
        execution_spec["distributed_spec"] = def_
        execution_spec["session_config"] = execution_spec.get("session_config")
        return execution_spec
    elif type_ == "multi-threaded":
        return execution_spec
    elif type_ == "single":
        return execution_spec

    if execution_spec.get('num_parallel') != None:
        assert type(execution_spec['num_parallel']) is int, "ERROR: num_parallel needs to be of type int but is of type {}!".format(type(execution_spec['num_parallel']).__name__)
        assert execution_spec['num_parallel'] > 0, "ERROR: num_parallel needs to be > 0 but is equal to {}".format(execution_spec['num_parallel'])
        return execution_spec

    raise TensorForceError("Unsupported execution type specified ({})!".format(type_))

Response: Sanity checks a execution - spec dict.


Instruction:
Write a Python 3 function that can
build and returns an Extraterrestrial Marauders game.

Response:
def make_game():
  """Builds and returns an Extraterrestrial Marauders game."""
  return ascii_art.ascii_art_to_game(
      GAME_ART, what_lies_beneath=' ',
      sprites=dict(
          [('P', PlayerSprite)] +
          [(c, UpwardLaserBoltSprite) for c in UPWARD_BOLT_CHARS] +
          [(c, DownwardLaserBoltSprite) for c in DOWNWARD_BOLT_CHARS]),
      drapes=dict(X=MarauderDrape,
                  B=BunkerDrape),
      update_schedule=['P', 'B', 'X'] + list(_ALL_BOLT_CHARS))


Instruction:
Here you have a function in Python 3, explain what it does
def _fly(self, board, layers, things, the_plot):
    # Disappear if we've hit a Marauder or a bunker.
    if (self.character in the_plot['bunker_hitters'] or
        self.character in the_plot['marauder_hitters']):
      return self._teleport((-1, -1))
    # Otherwise, northward!
    self._north(board, the_plot)

Response: Handles the behaviour of visible bolts flying toward Marauders.


Instruction:
Make a summary of the following Python 3 code
def _fire(self, layers, things, the_plot):
    # We don't fire if the player fired another bolt just now.
    if the_plot.get('last_player_shot') == the_plot.frame: return
    the_plot['last_player_shot'] = the_plot.frame
    # We start just above the player.
    row, col = things['P'].position
    self._teleport((row-1, col))

Response: Launches a new bolt from the player.


Instruction:
How would you explain what the following Python 3 function does
def _fly(self, board, layers, things, the_plot):
    # Disappear if we've hit a bunker.
    if self.character in the_plot['bunker_hitters']:
      return self._teleport((-1, -1))
    # End the game if we've hit the player.
    if self.position == things['P'].position: the_plot.terminate_episode()
    self._south(board, the_plot)

Response: Handles the behaviour of visible bolts flying toward the player.


Instruction:
Write a Python 3 function that can
launch a new bolt from a random Marauder.

Response:
def _fire(self, layers, the_plot):
    """Launches a new bolt from a random Marauder."""
    # We don't fire if another Marauder fired a bolt just now.
    if the_plot.get('last_marauder_shot') == the_plot.frame: return
    the_plot['last_marauder_shot'] = the_plot.frame
    # Which Marauder should fire the laser bolt?
    col = np.random.choice(np.nonzero(layers['X'].sum(axis=0))[0])
    row = np.nonzero(layers['X'][:, col])[0][-1] + 1
    # Move ourselves just below that Marauder.
    self._teleport((row, col))


Instruction:
Write a Python 3 function for
resetting the Runner s internal stats counters.

Response:
def reset(self, history=None):
        """
        Resets the Runner's internal stats counters.
        If history is empty, use default values in history.get().

        Args:
            history (dict): A dictionary containing an already run experiment's results. Keys should be:
                episode_rewards (list of rewards), episode_timesteps (lengths of episodes), episode_times (run-times)
        """
        if not history:
            history = dict()

        self.episode_rewards = history.get("episode_rewards", list())
        self.episode_timesteps = history.get("episode_timesteps", list())
        self.episode_times = history.get("episode_times", list())


Instruction:
Can you implement a function in Python 3 that
executes this runner by starting to act (via Agent(s)) in the given Environment(s). Stops execution according to certain conditions (e.g. max. number of episodes, etc..). Calls callback functions after each episode and/or after some summary criteria are met. Args: num_episodes (int): Max. number of episodes to run globally in total (across all threads/workers). num_timesteps (int): Max. number of time steps to run globally in total (across all threads/workers) max_episode_timesteps (int): Max. number of timesteps per episode. deterministic (bool): Whether to use exploration when selecting actions. episode_finished (callable): A function to be called once an episodes has finished. Should take a BaseRunner object and some worker ID (e.g. thread-ID or task-ID). Can decide for itself every how many episodes it should report something and what to report. summary_report (callable): Deprecated; Function that could produce a summary over the training progress so far. summary_interval (int): Deprecated; The number of time steps to execute (globally) before summary_report is called.

Response:
def run(self, num_episodes, num_timesteps, max_episode_timesteps, deterministic, episode_finished, summary_report,
            summary_interval):
        """
        Executes this runner by starting to act (via Agent(s)) in the given Environment(s).
        Stops execution according to certain conditions (e.g. max. number of episodes, etc..).
        Calls callback functions after each episode and/or after some summary criteria are met.

        Args:
            num_episodes (int): Max. number of episodes to run globally in total (across all threads/workers).
            num_timesteps (int): Max. number of time steps to run globally in total (across all threads/workers)
            max_episode_timesteps (int): Max. number of timesteps per episode.
            deterministic (bool): Whether to use exploration when selecting actions.
            episode_finished (callable): A function to be called once an episodes has finished. Should take
                a BaseRunner object and some worker ID (e.g. thread-ID or task-ID). Can decide for itself
                every how many episodes it should report something and what to report.
            summary_report (callable): Deprecated; Function that could produce a summary over the training
                progress so far.
            summary_interval (int): Deprecated; The number of time steps to execute (globally)
                before summary_report is called.
        """
        raise NotImplementedError


Instruction:
How would you code a function in Python 3 to
retrieve the experiences for given indices by combining entries from buffer elements which have no priorities and entries from priority memory.

Response:
def tf_retrieve_indices(self, buffer_elements, priority_indices):
        """
        Fetches experiences for given indices by combining entries from buffer
        which have no priorities, and entries from priority memory.

        Args:
            buffer_elements: Number of buffer elements to retrieve
            priority_indices: Index tensor for priority memory

        Returns: Batch of experiences
        """
        states = dict()
        buffer_start = self.buffer_index - buffer_elements
        buffer_end = self.buffer_index

        # Fetch entries from respective memories, concat.
        for name in sorted(self.states_memory):
            buffer_state_memory = self.states_buffer[name]
            # Slicing is more efficient than gathering, and buffer elements are always
            # fetched using contiguous indices.
            buffer_states = buffer_state_memory[buffer_start:buffer_end]
            # Memory indices are obtained via priority sampling, hence require gather.
            memory_states = tf.gather(params=self.states_memory[name], indices=priority_indices)
            states[name] = tf.concat(values=(buffer_states, memory_states), axis=0)

        internals = dict()
        for name in sorted(self.internals_memory):
            internal_buffer_memory = self.internals_buffer[name]
            buffer_internals = internal_buffer_memory[buffer_start:buffer_end]
            memory_internals = tf.gather(params=self.internals_memory[name], indices=priority_indices)
            internals[name] = tf.concat(values=(buffer_internals, memory_internals), axis=0)

        actions = dict()
        for name in sorted(self.actions_memory):
            action_buffer_memory = self.actions_buffer[name]
            buffer_action = action_buffer_memory[buffer_start:buffer_end]
            memory_action = tf.gather(params=self.actions_memory[name], indices=priority_indices)
            actions[name] = tf.concat(values=(buffer_action, memory_action), axis=0)

        buffer_terminal = self.terminal_buffer[buffer_start:buffer_end]
        priority_terminal = tf.gather(params=self.terminal_memory, indices=priority_indices)
        terminal = tf.concat(values=(buffer_terminal, priority_terminal), axis=0)

        buffer_reward = self.reward_buffer[buffer_start:buffer_end]
        priority_reward = tf.gather(params=self.reward_memory, indices=priority_indices)
        reward = tf.concat(values=(buffer_reward, priority_reward), axis=0)

        if self.include_next_states:
            assert util.rank(priority_indices) == 1
            next_priority_indices = (priority_indices + 1) % self.capacity
            next_buffer_start = (buffer_start + 1) % self.buffer_size
            next_buffer_end = (buffer_end + 1) % self.buffer_size

            next_states = dict()
            for name in sorted(self.states_memory):
                buffer_state_memory = self.states_buffer[name]
                buffer_next_states = buffer_state_memory[next_buffer_start:next_buffer_end]
                memory_next_states = tf.gather(params=self.states_memory[name], indices=next_priority_indices)
                next_states[name] = tf.concat(values=(buffer_next_states, memory_next_states), axis=0)

            next_internals = dict()
            for name in sorted(self.internals_memory):
                buffer_internal_memory = self.internals_buffer[name]
                buffer_next_internals = buffer_internal_memory[next_buffer_start:next_buffer_end]
                memory_next_internals = tf.gather(params=self.internals_memory[name], indices=next_priority_indices)
                next_internals[name] = tf.concat(values=(buffer_next_internals, memory_next_internals), axis=0)

            return dict(
                states=states,
                internals=internals,
                actions=actions,
                terminal=terminal,
                reward=reward,
                next_states=next_states,
                next_internals=next_internals
            )
        else:
            return dict(
                states=states,
                internals=internals,
                actions=actions,
                terminal=terminal,
                reward=reward
            )


Instruction:
Can you generate a brief explanation for the following Python 3 code
def tf_update_batch(self, loss_per_instance):
        # 1. We reconstruct the batch from the buffer and the priority memory via
        # the TensorFlow variables holding the respective indices.
        mask = tf.not_equal(
            x=self.batch_indices,
            y=tf.zeros(shape=tf.shape(input=self.batch_indices), dtype=tf.int32)
        )
        priority_indices = tf.reshape(tensor=tf.where(condition=mask), shape=[-1])

        # These are elements from the buffer which first need to be inserted into the main memory.
        sampled_buffer_batch = self.tf_retrieve_indices(
            buffer_elements=self.last_batch_buffer_elems,
            priority_indices=priority_indices
        )

        # Extract batch elements.
        states = sampled_buffer_batch['states']
        internals = sampled_buffer_batch['internals']
        actions = sampled_buffer_batch['actions']
        terminal = sampled_buffer_batch['terminal']
        reward = sampled_buffer_batch['reward']

        # 2. Compute priorities for all batch elements.
        priorities = loss_per_instance ** self.prioritization_weight
        assignments = list()

        # 3. Insert the buffer elements from the recent batch into memory,
        # overwrite memory if full.
        memory_end_index = self.memory_index + self.last_batch_buffer_elems
        memory_insert_indices = tf.range(
            start=self.memory_index,
            limit=memory_end_index
        ) % self.capacity

        for name in sorted(states):
            assignments.append(tf.scatter_update(
                ref=self.states_memory[name],
                indices=memory_insert_indices,
                # Only buffer elements from batch.
                updates=states[name][0:self.last_batch_buffer_elems])
            )
        for name in sorted(internals):
            assignments.append(tf.scatter_update(
                ref=self.internals_buffer[name],
                indices=memory_insert_indices,
                updates=internals[name][0:self.last_batch_buffer_elems]
            ))
        assignments.append(tf.scatter_update(
            ref=self.priorities,
            indices=memory_insert_indices,
            updates=priorities[0:self.last_batch_buffer_elems]
        ))
        assignments.append(tf.scatter_update(
            ref=self.terminal_memory,
            indices=memory_insert_indices,
            updates=terminal[0:self.last_batch_buffer_elems])
        )
        assignments.append(tf.scatter_update(
            ref=self.reward_memory,
            indices=memory_insert_indices,
            updates=reward[0:self.last_batch_buffer_elems])
        )
        for name in sorted(actions):
            assignments.append(tf.scatter_update(
                ref=self.actions_memory[name],
                indices=memory_insert_indices,
                updates=actions[name][0:self.last_batch_buffer_elems]
            ))

        # 4.Update the priorities of the elements already in the memory.
        # Slice out remaining elements - [] if all batch elements were from buffer.
        main_memory_priorities = priorities[self.last_batch_buffer_elems:]
        # Note that priority indices can have a different shape because multiple
        # samples can be from the same index.
        main_memory_priorities = main_memory_priorities[0:tf.shape(priority_indices)[0]]
        assignments.append(tf.scatter_update(
            ref=self.priorities,
            indices=priority_indices,
            updates=main_memory_priorities
        ))

        with tf.control_dependencies(control_inputs=assignments):
            # 5. Re-sort memory according to priorities.
            assignments = list()

            # Obtain sorted order and indices.
            sorted_priorities, sorted_indices = tf.nn.top_k(
                input=self.priorities,
                k=self.capacity,
                sorted=True
            )
            # Re-assign elements according to priorities.
            # Priorities was the tensor we used to sort, so this can be directly assigned.
            assignments.append(tf.assign(ref=self.priorities, value=sorted_priorities))

            # All other memory variables are assigned via scatter updates using the indices
            # returned by the sort:
            assignments.append(tf.scatter_update(
                ref=self.terminal_memory,
                indices=sorted_indices,
                updates=self.terminal_memory
            ))
            for name in sorted(self.states_memory):
                assignments.append(tf.scatter_update(
                    ref=self.states_memory[name],
                    indices=sorted_indices,
                    updates=self.states_memory[name]
                ))
            for name in sorted(self.actions_memory):
                assignments.append(tf.scatter_update(
                    ref=self.actions_memory[name],
                    indices=sorted_indices,
                    updates=self.actions_memory[name]
                ))
            for name in sorted(self.internals_memory):
                assignments.append(tf.scatter_update(
                    ref=self.internals_memory[name],
                    indices=sorted_indices,
                    updates=self.internals_memory[name]
                ))
            assignments.append(tf.scatter_update(
                ref=self.reward_memory,
                indices=sorted_indices,
                updates=self.reward_memory
            ))

        # 6. Reset buffer index and increment memory index by inserted elements.
        with tf.control_dependencies(control_inputs=assignments):
            assignments = list()
            # Decrement pointer of last elements used.
            assignments.append(tf.assign_sub(ref=self.buffer_index, value=self.last_batch_buffer_elems))

            # Keep track of memory size as to know whether we can sample from the main memory.
            # Since the memory pointer can set to 0, we want to know if we are at capacity.
            total_inserted_elements = self.memory_size + self.last_batch_buffer_elems
            assignments.append(tf.assign(
                ref=self.memory_size,
                value=tf.minimum(x=total_inserted_elements, y=self.capacity))
            )

            # Update memory insertion index.
            assignments.append(tf.assign(ref=self.memory_index, value=memory_end_index))

            # Reset batch indices.
            assignments.append(tf.assign(
                ref=self.batch_indices,
                value=tf.zeros(shape=tf.shape(self.batch_indices), dtype=tf.int32)
            ))

        with tf.control_dependencies(control_inputs=assignments):
            return tf.no_op()

Response: Update the priority memory for the current batch.


Instruction:
Can you generate the documentation for the following Python 3 function
def setup_components_and_tf_funcs(self, custom_getter=None):
        # Create network before super-call, since non-empty internals_spec attribute (for RNN) is required subsequently.
        self.network = Network.from_spec(
            spec=self.network_spec,
            kwargs=dict(summary_labels=self.summary_labels)
        )

        # Now that we have the network component: We can create the internals placeholders.
        assert len(self.internals_spec) == 0
        self.internals_spec = self.network.internals_spec()
        for name in sorted(self.internals_spec):
            internal = self.internals_spec[name]
            self.internals_input[name] = tf.placeholder(
                dtype=util.tf_dtype(internal['type']),
                shape=(None,) + tuple(internal['shape']),
                name=('internal-' + name)
            )
            if internal['initialization'] == 'zeros':
                self.internals_init[name] = np.zeros(shape=internal['shape'])
            else:
                raise TensorForceError("Invalid internal initialization value.")

        # And only then call super.
        custom_getter = super(DistributionModel, self).setup_components_and_tf_funcs(custom_getter)

        # Distributions
        self.distributions = self.create_distributions()

        # KL divergence function
        self.fn_kl_divergence = tf.make_template(
            name_='kl-divergence',
            func_=self.tf_kl_divergence,
            custom_getter_=custom_getter
        )

        return custom_getter

Response: Creates and stores Network and Distribution objects.


Instruction:
Can you write a function in Python 3 where it
creates and returns the distributions according to self. distributions_spec.

Response:
def create_distributions(self):
        """
        Creates and returns the Distribution objects based on self.distributions_spec.

        Returns: Dict of distributions according to self.distributions_spec.
        """
        distributions = dict()
        for name in sorted(self.actions_spec):
            action = self.actions_spec[name]

            if self.distributions_spec is not None and name in self.distributions_spec:
                kwargs = dict(action)
                kwargs['scope'] = name
                kwargs['summary_labels'] = self.summary_labels
                distributions[name] = Distribution.from_spec(
                    spec=self.distributions_spec[name],
                    kwargs=kwargs
                )

            elif action['type'] == 'bool':
                distributions[name] = Bernoulli(
                    shape=action['shape'],
                    scope=name,
                    summary_labels=self.summary_labels
                )

            elif action['type'] == 'int':
                distributions[name] = Categorical(
                    shape=action['shape'],
                    num_actions=action['num_actions'],
                    scope=name,
                    summary_labels=self.summary_labels
                )

            elif action['type'] == 'float':
                if 'min_value' in action:
                    distributions[name] = Beta(
                        shape=action['shape'],
                        min_value=action['min_value'],
                        max_value=action['max_value'],
                        scope=name,
                        summary_labels=self.summary_labels
                    )

                else:
                    distributions[name] = Gaussian(
                        shape=action['shape'],
                        scope=name,
                        summary_labels=self.summary_labels
                    )

        return distributions


Instruction:
Can you generate a brief explanation for the following Python 3 code
def from_spec(spec):
        exploration = util.get_object(
            obj=spec,
            predefined_objects=tensorforce.core.explorations.explorations
        )
        assert isinstance(exploration, Exploration)
        return exploration

Response: Creates an exploration object from a specification dict.


Instruction:
Can you write a function in Python 3 where it
creates a Memory object from a specification dict.

Response:
def from_spec(spec, kwargs=None):
        """
        Creates a memory from a specification dict.
        """
        memory = util.get_object(
            obj=spec,
            predefined_objects=tensorforce.core.memories.memories,
            kwargs=kwargs
        )
        assert isinstance(memory, Memory)
        return memory


Instruction:
Can you implement a function in Python 3 that
creates the TensorFlow operations for performing an optimization step. Args: time: Time tensor. variables: List of variables to optimize. arguments: Dict of arguments for callables, like fn_loss. **kwargs: Additional arguments passed on to the internal optimizer. Returns: List of delta tensors corresponding to the updates for each optimized variable.

Response:
def tf_step(
        self,
        time,
        variables,
        arguments,
        **kwargs
    ):
        """
        Creates the TensorFlow operations for performing an optimization step.

        Args:
            time: Time tensor.
            variables: List of variables to optimize.
            arguments: Dict of arguments for callables, like fn_loss.
            **kwargs: Additional arguments passed on to the internal optimizer.

        Returns:
            List of delta tensors corresponding to the updates for each optimized variable.
        """
        # Get some (batched) argument to determine batch size.
        arguments_iter = iter(arguments.values())
        some_argument = next(arguments_iter)

        try:
            while not isinstance(some_argument, tf.Tensor) or util.rank(some_argument) == 0:
                if isinstance(some_argument, dict):
                    if some_argument:
                        arguments_iter = iter(some_argument.values())
                    some_argument = next(arguments_iter)
                elif isinstance(some_argument, list):
                    if some_argument:
                        arguments_iter = iter(some_argument)
                    some_argument = next(arguments_iter)
                elif some_argument is None or util.rank(some_argument) == 0:
                    # Non-batched argument
                    some_argument = next(arguments_iter)
                else:
                    raise TensorForceError("Invalid argument type.")
        except StopIteration:
            raise TensorForceError("Invalid argument type.")

        batch_size = tf.shape(input=some_argument)[0]
        num_samples = tf.cast(
            x=(self.fraction * tf.cast(x=batch_size, dtype=util.tf_dtype('float'))),
            dtype=util.tf_dtype('int')
        )
        num_samples = tf.maximum(x=num_samples, y=1)
        indices = tf.random_uniform(shape=(num_samples,), maxval=batch_size, dtype=tf.int32)

        subsampled_arguments = util.map_tensors(
            fn=(lambda arg: arg if util.rank(arg) == 0 else tf.gather(params=arg, indices=indices)),
            tensors=arguments
        )

        return self.optimizer.step(
            time=time,
            variables=variables,
            arguments=subsampled_arguments,
            **kwargs
        )


Instruction:
Can you tell what is the following Python 3 function doing
def run(self, num_timesteps=None, num_episodes=None, max_episode_timesteps=None, deterministic=False,
            episode_finished=None, summary_report=None, summary_interval=None, timesteps=None, episodes=None, testing=False, sleep=None
            ):
        """
        Args:
            timesteps (int): Deprecated; see num_timesteps.
            episodes (int): Deprecated; see num_episodes.
        """

        # deprecation warnings
        if timesteps is not None:
            num_timesteps = timesteps
            warnings.warn("WARNING: `timesteps` parameter is deprecated, use `num_timesteps` instead.",
                          category=DeprecationWarning)
        if episodes is not None:
            num_episodes = episodes
            warnings.warn("WARNING: `episodes` parameter is deprecated, use `num_episodes` instead.",
                          category=DeprecationWarning)

        # figure out whether we are using the deprecated way of "episode_finished" reporting
        old_episode_finished = False
        if episode_finished is not None and len(getargspec(episode_finished).args) == 1:
            old_episode_finished = True

        # Keep track of episode reward and episode length for statistics.
        self.start_time = time.time()

        self.agent.reset()

        if num_episodes is not None:
            num_episodes += self.agent.episode

        if num_timesteps is not None:
            num_timesteps += self.agent.timestep

        # add progress bar
        with tqdm(total=num_episodes) as pbar:
            # episode loop
            while True:
                episode_start_time = time.time()
                state = self.environment.reset()
                self.agent.reset()

                # Update global counters.
                self.global_episode = self.agent.episode  # global value (across all agents)
                self.global_timestep = self.agent.timestep  # global value (across all agents)

                episode_reward = 0
                self.current_timestep = 0

                # time step (within episode) loop
                while True:
                    action = self.agent.act(states=state, deterministic=deterministic)

                    reward = 0
                    for _ in xrange(self.repeat_actions):
                        state, terminal, step_reward = self.environment.execute(action=action)
                        reward += step_reward
                        if terminal:
                            break

                    if max_episode_timesteps is not None and self.current_timestep >= max_episode_timesteps:
                        terminal = True

                    if not testing:
                        self.agent.observe(terminal=terminal, reward=reward)

                    self.global_timestep += 1
                    self.current_timestep += 1
                    episode_reward += reward

                    if terminal or self.agent.should_stop():  # TODO: should_stop also terminate?
                        break

                    if sleep is not None:
                        time.sleep(sleep)

                # Update our episode stats.
                time_passed = time.time() - episode_start_time
                self.episode_rewards.append(episode_reward)
                self.episode_timesteps.append(self.current_timestep)
                self.episode_times.append(time_passed)

                self.global_episode += 1
                pbar.update(1)

                # Check, whether we should stop this run.
                if episode_finished is not None:
                    # deprecated way (passing in only runner object):
                    if old_episode_finished:
                        if not episode_finished(self):
                            break
                    # new unified way (passing in BaseRunner AND some worker ID):
                    elif not episode_finished(self, self.id):
                        break
                if (num_episodes is not None and self.global_episode >= num_episodes) or \
                        (num_timesteps is not None and self.global_timestep >= num_timesteps) or \
                        self.agent.should_stop():
                    break
            pbar.update(num_episodes - self.global_episode)

Response: Runs the main loop of the system.


Instruction:
Write a Python 3 script to
retrieve the list of experiences for given indices.

Response:
def tf_retrieve_indices(self, indices):
        """
        Fetches experiences for given indices.

        Args:
            indices: Index tensor

        Returns: Batch of experiences
        """
        states = dict()
        for name in sorted(self.states_memory):
            states[name] = tf.gather(params=self.states_memory[name], indices=indices)

        internals = dict()
        for name in sorted(self.internals_memory):
            internals[name] = tf.gather(params=self.internals_memory[name], indices=indices)

        actions = dict()
        for name in sorted(self.actions_memory):
            actions[name] = tf.gather(params=self.actions_memory[name], indices=indices)

        terminal = tf.gather(params=self.terminal_memory, indices=indices)
        reward = tf.gather(params=self.reward_memory, indices=indices)

        if self.include_next_states:
            assert util.rank(indices) == 1
            next_indices = (indices + 1) % self.capacity

            next_states = dict()
            for name in sorted(self.states_memory):
                next_states[name] = tf.gather(params=self.states_memory[name], indices=next_indices)

            next_internals = dict()
            for name in sorted(self.internals_memory):
                next_internals[name] = tf.gather(params=self.internals_memory[name], indices=next_indices)

            return dict(
                states=states,
                internals=internals,
                actions=actions,
                terminal=terminal,
                reward=reward,
                next_states=next_states,
                next_internals=next_internals
            )
        else:
            return dict(
                states=states,
                internals=internals,
                actions=actions,
                terminal=terminal,
                reward=reward
            )


Instruction:
How would you explain what the following Python 3 function does
def tf_solve(self, fn_x, x_init, base_value, target_value, estimated_improvement=None):
        return super(LineSearch, self).tf_solve(fn_x, x_init, base_value, target_value, estimated_improvement)

Response: Iteratively optimizes the value at x_0 at x_1 and returns the solution to the problem.


Instruction:
Write a Python 3 script for
initializing the term for the current iteration.

Response:
def tf_initialize(self, x_init, base_value, target_value, estimated_improvement):
        """
        Initialization step preparing the arguments for the first iteration of the loop body.

        Args:
            x_init: Initial solution guess $x_0$.
            base_value: Value $f(x')$ at $x = x'$.
            target_value: Value $f(x_0)$ at $x = x_0$.
            estimated_improvement: Estimated value at $x = x_0$, $f(x')$ if None.

        Returns:
            Initial arguments for tf_step.
        """
        self.base_value = base_value

        if estimated_improvement is None:  # TODO: Is this a good alternative?
            estimated_improvement = tf.abs(x=base_value)

        first_step = super(LineSearch, self).tf_initialize(x_init)

        improvement = tf.divide(
            x=(target_value - self.base_value),
            y=tf.maximum(x=estimated_improvement, y=util.epsilon)
        )

        last_improvement = improvement - 1.0

        if self.mode == 'linear':
            deltas = [-t * self.parameter for t in x_init]
            self.estimated_incr = -estimated_improvement * self.parameter

        elif self.mode == 'exponential':
            deltas = [-t * self.parameter for t in x_init]

        return first_step + (deltas, improvement, last_improvement, estimated_improvement)


Instruction:
How would you explain what the following Python 3 function does
def tf_step(self, x, iteration, deltas, improvement, last_improvement, estimated_improvement):
        x, next_iteration, deltas, improvement, last_improvement, estimated_improvement = super(LineSearch, self).tf_step(
            x, iteration, deltas, improvement, last_improvement, estimated_improvement
        )

        next_x = [t + delta for t, delta in zip(x, deltas)]

        if self.mode == 'linear':
            next_deltas = deltas
            next_estimated_improvement = estimated_improvement + self.estimated_incr

        elif self.mode == 'exponential':
            next_deltas = [delta * self.parameter for delta in deltas]
            next_estimated_improvement = estimated_improvement * self.parameter

        target_value = self.fn_x(next_deltas)

        next_improvement = tf.divide(
            x=(target_value - self.base_value),
            y=tf.maximum(x=next_estimated_improvement, y=util.epsilon)
        )

        return next_x, next_iteration, next_deltas, next_improvement, improvement, next_estimated_improvement

Response: This method is called by the iteration loop body of the line search algorithm.


Instruction:
Can you generate the documentation for the following Python 3 function
def tf_next_step(self, x, iteration, deltas, improvement, last_improvement, estimated_improvement):
        next_step = super(LineSearch, self).tf_next_step(
            x, iteration, deltas, improvement, last_improvement, estimated_improvement
        )

        def undo_deltas():
            value = self.fn_x([-delta for delta in deltas])
            with tf.control_dependencies(control_inputs=(value,)):
                # Trivial operation to enforce control dependency
                return tf.less(x=value, y=value)  # == False

        improved = tf.cond(
            pred=(improvement > last_improvement),
            true_fn=(lambda: True),
            false_fn=undo_deltas
        )

        next_step = tf.logical_and(x=next_step, y=improved)
        next_step = tf.logical_and(x=next_step, y=(improvement < self.accept_ratio))
        return tf.logical_and(x=next_step, y=(estimated_improvement > util.epsilon))

Response: Returns a term that checks if another iteration should be performed.


Instruction:
Write a Python 3 function that can
escape special characters &lt and &gt to HTML - safe sequences.

Response:
def escape(text, quote=False, smart_amp=True):
    """Replace special characters "&", "<" and ">" to HTML-safe sequences.

    The original cgi.escape will always escape "&", but you can control
    this one for a smart escape amp.

    :param quote: if set to True, " and ' will be escaped.
    :param smart_amp: if set to False, & will always be escaped.
    """
    if smart_amp:
        text = _escape_pattern.sub('&', text)
    else:
        text = text.replace('&', '&')
    text = text.replace('<', '<')
    text = text.replace('>', '>')
    if quote:
        text = text.replace('"', '"')
        text = text.replace("'", ''')
    return text


Instruction:
Write a Python 3 function that can
remove dangerous URL schemes like javascript: and escape afterwards.

Response:
def escape_link(url):
    """Remove dangerous URL schemes like javascript: and escape afterwards."""
    lower_url = url.lower().strip('\x00\x1a \n\r\t')
    for scheme in _scheme_blacklist:
        if lower_url.startswith(scheme):
            return ''
    return escape(url, quote=True, smart_amp=False)


Instruction:
Can you generate the documentation for the following Python 3 function
def markdown(text, escape=True, **kwargs):
    return Markdown(escape=escape, **kwargs)(text)

Response: Render markdown formatted text to html.


Instruction:
Can you generate the documentation for the following Python 3 function
def hard_wrap(self):
        self.linebreak = re.compile(r'^ *\n(?!\s*$)')
        self.text = re.compile(
            r'^[\s\S]+?(?=[\\<!\[_*`~]|https?://| *\n|$)'
        )

Response: Grammar for hard wrap linebreak.


Instruction:
Implement a Python 3 function for
rendering block level code.

Response:
def block_code(self, code, lang=None):
        """Rendering block level code. ``pre > code``.

        :param code: text content of the code block.
        :param lang: language of the given code.
        """
        code = code.rstrip('\n')
        if not lang:
            code = escape(code, smart_amp=False)
            return '
%s\n
\n' % code
        code = escape(code, quote=True, smart_amp=False)
        return '
%s\n
\n' % (lang, code)


Instruction:
Write a Python 3 function that can
render block level pure html content.

Response:
def block_html(self, html):
        """Rendering block level pure html content.

        :param html: text content of the html snippet.
        """
        if self.options.get('skip_style') and \
           html.lower().startswith('<style'):
            return ''
        if self.options.get('escape'):
            return escape(html)
        return html


Instruction:
Explain what the following Python 3 code does
def list(self, body, ordered=True):
        tag = 'ul'
        if ordered:
            tag = 'ol'
        return '<%s>\n%s</%s>\n' % (tag, body, tag)

Response: Render list tags like ul and ol.


Instruction:
Implement a function in Python 3 to
render a table cell. Like a table cell. Like a table cell.

Response:
def table_cell(self, content, **flags):
        """Rendering a table cell. Like ``<th>`` ``<td>``.

        :param content: content of current table cell.
        :param header: whether this is header or not.
        :param align: align of current table cell.
        """
        if flags['header']:
            tag = 'th'
        else:
            tag = 'td'
        align = flags['align']
        if not align:
            return '<%s>%s</%s>\n' % (tag, content, tag)
        return '<%s style="text-align:%s">%s</%s>\n' % (
            tag, align, content, tag
        )


Instruction:
Make a summary of the following Python 3 code
def codespan(self, text):
        text = escape(text.rstrip(), smart_amp=False)
        return '<code>%s</code>' % text

Response: Rendering inline code text.


Instruction:
Given the following Python 3 function, write the documentation
def autolink(self, link, is_email=False):
        text = link = escape(link)
        if is_email:
            link = 'mailto:%s' % link
        return '%s' % (link, text)

Response: Render a given link or email address.


Instruction:
How would you explain what the following Python 3 function does
def link(self, link, title, text):
        link = escape_link(link)
        if not title:
            return '%s' % (link, text)
        title = escape(title, quote=True)
        return '%s' % (link, title, text)

Response: Rendering a given link with content and title.


Instruction:
Create a Python 3 function to
render a image with title and text.

Response:
def image(self, src, title, text):
        """Rendering a image with title and text.

        :param src: source link of the image.
        :param title: title text of the image.
        :param text: alt text of the image.
        """
        src = escape_link(src)
        text = escape(text, quote=True)
        if title:
            title = escape(title, quote=True)
            html = '<img src="%s" alt="%s" title="%s"' % (src, text, title)
        else:
            html = '<img src="%s" alt="%s"' % (src, text)
        if self.options.get('use_xhtml'):
            return '%s />' % html
        return '%s>' % html


Instruction:
Here you have a function in Python 3, explain what it does
def footnote_ref(self, key, index):
        html = (
            ''
            '%d'
        ) % (escape(key), escape(key), index)
        return html

Response: Renders the ref anchor of a footnote.


Instruction:
Write a Python 3 function that can
render a footnote item.

Response:
def footnote_item(self, key, text):
        """Rendering a footnote item.

        :param key: identity key for the footnote.
        :param text: text content of the footnote.
        """
        back = (
            '↩'
        ) % escape(key)
        text = text.rstrip()
        if text.endswith('
'):
            text = re.sub(r'<\/p>$', r'%s
' % back, text)
        else:
            text = '%s
%s
' % (text, back)
        html = '
%s
\n' % (escape(key), text)
        return html


Instruction:
Make a summary of the following Python 3 code
def build_metagraph_list(self):
        ops = []

        self.ignore_unknown_dtypes = True
        for key in sorted(self.meta_params):
            value = self.convert_data_to_string(self.meta_params[key])

            if len(value) == 0:
                continue
            if isinstance(value, str):
                ops.append(tf.contrib.summary.generic(name=key, tensor=tf.convert_to_tensor(str(value))))
            else:
                ops.append(tf.contrib.summary.generic(name=key, tensor=tf.as_string(tf.convert_to_tensor(value))))

        return ops

Response: Convert MetaParams into TF Summary Format and create summary_op.


Instruction:
Write a Python 3 script for
enabling markdown syntax in docstrings

Response:
def process_docstring(app, what, name, obj, options, lines):
    """Enable markdown syntax in docstrings"""

    markdown = "\n".join(lines)

    # ast = cm_parser.parse(markdown)
    # html = cm_renderer.render(ast)
    rest = m2r(markdown)

    rest.replace("\r\n", "\n")
    del lines[:]
    lines.extend(rest.split("\n"))


Instruction:
Create a Python 3 function to
create the TensorFlow operations for performing an optimization step. Args: time: Time tensor. variables: List of variables to optimize. arguments: Dict of arguments for callables, like fn_loss. fn_loss: A callable returning the loss of the current model. fn_kl_divergence: A callable returning the KL-divergence relative to the current model. return_estimated_improvement: Returns the estimated improvement resulting from the natural gradient calculation if true. **kwargs: Additional arguments, not used. Returns: List of delta tensors corresponding to the updates for each optimized variable.

Response:
def tf_step(
        self,
        time,
        variables,
        arguments,
        fn_loss,
        fn_kl_divergence,
        return_estimated_improvement=False,
        **kwargs
    ):
        """
        Creates the TensorFlow operations for performing an optimization step.

        Args:
            time: Time tensor.
            variables: List of variables to optimize.
            arguments: Dict of arguments for callables, like fn_loss.
            fn_loss: A callable returning the loss of the current model.
            fn_kl_divergence: A callable returning the KL-divergence relative to the current model.
            return_estimated_improvement: Returns the estimated improvement resulting from the
                natural gradient calculation if true.
            **kwargs: Additional arguments, not used.

        Returns:
            List of delta tensors corresponding to the updates for each optimized variable.
        """

        # Optimize: argmin(w) loss(w + delta) such that kldiv(P(w) || P(w + delta)) = learning_rate
        # For more details, see our blogpost:
        # https://reinforce.io/blog/end-to-end-computation-graphs-for-reinforcement-learning/

        # from tensorforce import util
        # arguments = util.map_tensors(fn=tf.stop_gradient, tensors=arguments)

        # kldiv
        kldiv = fn_kl_divergence(**arguments)

        # grad(kldiv)
        kldiv_gradients = tf.gradients(ys=kldiv, xs=variables)

        # Calculates the product x * F of a given vector x with the fisher matrix F.
        # Incorporating the product prevents having to calculate the entire matrix explicitly.
        def fisher_matrix_product(deltas):
            # Gradient is not propagated through solver.
            deltas = [tf.stop_gradient(input=delta) for delta in deltas]

            # delta' * grad(kldiv)
            delta_kldiv_gradients = tf.add_n(inputs=[
                tf.reduce_sum(input_tensor=(delta * grad)) for delta, grad in zip(deltas, kldiv_gradients)
            ])

            # [delta' * F] = grad(delta' * grad(kldiv))
            return tf.gradients(ys=delta_kldiv_gradients, xs=variables)

        # loss
        loss = fn_loss(**arguments)

        # grad(loss)
        loss_gradients = tf.gradients(ys=loss, xs=variables)

        # Solve the following system for delta' via the conjugate gradient solver.
        # [delta' * F] * delta' = -grad(loss)
        # --> delta'  (= lambda * delta)
        deltas = self.solver.solve(fn_x=fisher_matrix_product, x_init=None, b=[-grad for grad in loss_gradients])

        # delta' * F
        delta_fisher_matrix_product = fisher_matrix_product(deltas=deltas)

        # c' = 0.5 * delta' * F * delta'  (= lambda * c)
        # TODO: Why constant and hence KL-divergence sometimes negative?
        constant = 0.5 * tf.add_n(inputs=[
            tf.reduce_sum(input_tensor=(delta_F * delta))
            for delta_F, delta in zip(delta_fisher_matrix_product, deltas)
        ])

        # Natural gradient step if constant > 0
        def natural_gradient_step():
            # lambda = sqrt(c' / c)
            lagrange_multiplier = tf.sqrt(x=(constant / self.learning_rate))

            # delta = delta' / lambda
            estimated_deltas = [delta / lagrange_multiplier for delta in deltas]

            # improvement = grad(loss) * delta  (= loss_new - loss_old)
            estimated_improvement = tf.add_n(inputs=[
                tf.reduce_sum(input_tensor=(grad * delta))
                for grad, delta in zip(loss_gradients, estimated_deltas)
            ])

            # Apply natural gradient improvement.
            applied = self.apply_step(variables=variables, deltas=estimated_deltas)

            with tf.control_dependencies(control_inputs=(applied,)):
                # Trivial operation to enforce control dependency
                if return_estimated_improvement:
                    return [estimated_delta + 0.0 for estimated_delta in estimated_deltas], estimated_improvement
                else:
                    return [estimated_delta + 0.0 for estimated_delta in estimated_deltas]

        # Zero step if constant <= 0
        def zero_step():
            if return_estimated_improvement:
                return [tf.zeros_like(tensor=delta) for delta in deltas], 0.0
            else:
                return [tf.zeros_like(tensor=delta) for delta in deltas]

        # Natural gradient step only works if constant > 0
        return tf.cond(pred=(constant > 0.0), true_fn=natural_gradient_step, false_fn=zero_step)


Instruction:
How would you explain what the following Python 3 function does
def tf_step(self, time, variables, arguments, fn_reference=None, **kwargs):

        # Set reference to compare with at each optimization step, in case of a comparative loss.
        arguments['reference'] = fn_reference(**arguments)

        # First step
        deltas = self.optimizer.step(time=time, variables=variables, arguments=arguments, **kwargs)

        if self.unroll_loop:
            # Unrolled for loop
            for _ in xrange(self.num_steps - 1):
                with tf.control_dependencies(control_inputs=deltas):
                    step_deltas = self.optimizer.step(time=time, variables=variables, arguments=arguments, **kwargs)
                    deltas = [delta1 + delta2 for delta1, delta2 in zip(deltas, step_deltas)]

            return deltas

        else:
            # TensorFlow while loop
            def body(iteration, deltas):
                with tf.control_dependencies(control_inputs=deltas):
                    step_deltas = self.optimizer.step(time=time, variables=variables, arguments=arguments, **kwargs)
                    deltas = [delta1 + delta2 for delta1, delta2 in zip(deltas, step_deltas)]
                    return iteration + 1, deltas

            def cond(iteration, deltas):
                return iteration < self.num_steps - 1

            _, deltas = tf.while_loop(cond=cond, body=body, loop_vars=(0, deltas))

            return deltas

Response: Creates the TensorFlow operations for performing an optimization step.


Instruction:
Implement a Python 3 function for
returning the TensorFlow operations for calculating the baseline loss of a batch.

Response:
def tf_baseline_loss(self, states, internals, reward, update, reference=None):
        """
        Creates the TensorFlow operations for calculating the baseline loss of a batch.

        Args:
            states: Dict of state tensors.
            internals: List of prior internal state tensors.
            reward: Reward tensor.
            update: Boolean tensor indicating whether this call happens during an update.
            reference: Optional reference tensor(s), in case of a comparative loss.

        Returns:
            Loss tensor.
        """
        if self.baseline_mode == 'states':
            loss = self.baseline.loss(
                states=states,
                internals=internals,
                reward=reward,
                update=update,
                reference=reference
            )

        elif self.baseline_mode == 'network':
            loss = self.baseline.loss(
                states=self.network.apply(x=states, internals=internals, update=update),
                internals=internals,
                reward=reward,
                update=update,
                reference=reference
            )

        regularization_loss = self.baseline.regularization_loss()
        if regularization_loss is not None:
            loss += regularization_loss

        return loss


Instruction:
Can you create a Python 3 function that
returns the baseline optimizer arguments as a dict.

Response:
def baseline_optimizer_arguments(self, states, internals, reward):
        """
        Returns the baseline optimizer arguments including the time, the list of variables to
        optimize, and various functions which the optimizer might require to perform an update
        step.

        Args:
            states: Dict of state tensors.
            internals: List of prior internal state tensors.
            reward: Reward tensor.

        Returns:
            Baseline optimizer arguments as dict.
        """
        arguments = dict(
            time=self.global_timestep,
            variables=self.baseline.get_variables(),
            arguments=dict(
                states=states,
                internals=internals,
                reward=reward,
                update=tf.constant(value=True),
            ),
            fn_reference=self.baseline.reference,
            fn_loss=self.fn_baseline_loss,
            # source_variables=self.network.get_variables()
        )
        if self.global_model is not None:
            arguments['global_variables'] = self.global_model.baseline.get_variables()
        return arguments


Instruction:
Explain what the following Python 3 code does
def tf_step(self, time, variables, source_variables, **kwargs):
        assert all(util.shape(source) == util.shape(target) for source, target in zip(source_variables, variables))

        last_sync = tf.get_variable(
            name='last-sync',
            shape=(),
            dtype=tf.int64,
            initializer=tf.constant_initializer(value=(-self.sync_frequency), dtype=tf.int64),
            trainable=False
        )

        def sync():
            deltas = list()
            for source_variable, target_variable in zip(source_variables, variables):
                delta = self.update_weight * (source_variable - target_variable)
                deltas.append(delta)

            applied = self.apply_step(variables=variables, deltas=deltas)
            last_sync_updated = last_sync.assign(value=time)

            with tf.control_dependencies(control_inputs=(applied, last_sync_updated)):
                # Trivial operation to enforce control dependency
                return [delta + 0.0 for delta in deltas]

        def no_sync():
            deltas = list()
            for variable in variables:
                delta = tf.zeros(shape=util.shape(variable))
                deltas.append(delta)
            return deltas

        do_sync = (time - last_sync >= self.sync_frequency)
        return tf.cond(pred=do_sync, true_fn=sync, false_fn=no_sync)

Response: Creates a TensorFlow operation for performing an optimization step.


Instruction:
Implement a function in Python 3 to
create the TensorFlow operations for performing an optimization step. Args: time: Time tensor. variables: List of variables to optimize. arguments: Dict of arguments for callables, like fn_loss. fn_loss: A callable returning the loss of the current model. **kwargs: Additional arguments, not used. Returns: List of delta tensors corresponding to the updates for each optimized variable.

Response:
def tf_step(
        self,
        time,
        variables,
        arguments,
        fn_loss,
        **kwargs
    ):
        """
        Creates the TensorFlow operations for performing an optimization step.

        Args:
            time: Time tensor.
            variables: List of variables to optimize.
            arguments: Dict of arguments for callables, like fn_loss.
            fn_loss: A callable returning the loss of the current model.
            **kwargs: Additional arguments, not used.

        Returns:
            List of delta tensors corresponding to the updates for each optimized variable.
        """
        unperturbed_loss = fn_loss(**arguments)

        # First sample
        perturbations = [tf.random_normal(shape=util.shape(variable)) * self.learning_rate for variable in variables]
        applied = self.apply_step(variables=variables, deltas=perturbations)

        with tf.control_dependencies(control_inputs=(applied,)):
            perturbed_loss = fn_loss(**arguments)
            direction = tf.sign(x=(unperturbed_loss - perturbed_loss))
            deltas_sum = [direction * perturbation for perturbation in perturbations]

        if self.unroll_loop:
            # Unrolled for loop
            previous_perturbations = perturbations
            for sample in xrange(self.num_samples):

                with tf.control_dependencies(control_inputs=deltas_sum):
                    perturbations = [tf.random_normal(shape=util.shape(variable)) * self.learning_rate for variable in variables]
                    perturbation_deltas = [
                        pert - prev_pert for pert, prev_pert in zip(perturbations, previous_perturbations)
                    ]
                    applied = self.apply_step(variables=variables, deltas=perturbation_deltas)
                    previous_perturbations = perturbations

                with tf.control_dependencies(control_inputs=(applied,)):
                    perturbed_loss = fn_loss(**arguments)
                    direction = tf.sign(x=(unperturbed_loss - perturbed_loss))
                    deltas_sum = [delta + direction * perturbation for delta, perturbation in zip(deltas_sum, perturbations)]

        else:
            # TensorFlow while loop
            def body(iteration, deltas_sum, previous_perturbations):

                with tf.control_dependencies(control_inputs=deltas_sum):
                    perturbations = [tf.random_normal(shape=util.shape(variable)) * self.learning_rate for variable in variables]
                    perturbation_deltas = [
                        pert - prev_pert for pert, prev_pert in zip(perturbations, previous_perturbations)
                    ]
                    applied = self.apply_step(variables=variables, deltas=perturbation_deltas)

                with tf.control_dependencies(control_inputs=(applied,)):
                    perturbed_loss = fn_loss(**arguments)
                    direction = tf.sign(x=(unperturbed_loss - perturbed_loss))
                    deltas_sum = [delta + direction * perturbation for delta, perturbation in zip(deltas_sum, perturbations)]

                return iteration + 1, deltas_sum, perturbations

            def cond(iteration, deltas_sum, previous_perturbation):
                return iteration < self.num_samples - 1

            _, deltas_sum, perturbations = tf.while_loop(cond=cond, body=body, loop_vars=(0, deltas_sum, perturbations))

        with tf.control_dependencies(control_inputs=deltas_sum):
            deltas = [delta / self.num_samples for delta in deltas_sum]
            perturbation_deltas = [delta - pert for delta, pert in zip(deltas, perturbations)]
            applied = self.apply_step(variables=variables, deltas=perturbation_deltas)

        with tf.control_dependencies(control_inputs=(applied,)):
            # Trivial operation to enforce control dependency
            return [delta + 0.0 for delta in deltas]


Instruction:
Explain what the following Python 3 code does
def tf_step(self, time, variables, **kwargs):

        global_variables = kwargs["global_variables"]

        assert all(
            util.shape(global_variable) == util.shape(local_variable)
            for global_variable, local_variable in zip(global_variables, variables)
        )

        local_deltas = self.optimizer.step(time=time, variables=variables, **kwargs)

        with tf.control_dependencies(control_inputs=local_deltas):
            applied = self.optimizer.apply_step(variables=global_variables, deltas=local_deltas)

        with tf.control_dependencies(control_inputs=(applied,)):
            update_deltas = list()
            for global_variable, local_variable in zip(global_variables, variables):
                delta = global_variable - local_variable
                update_deltas.append(delta)

            applied = self.apply_step(variables=variables, deltas=update_deltas)

            # TODO: Update time, episode, etc (like in Synchronization)?

        with tf.control_dependencies(control_inputs=(applied,)):
            return [local_delta + update_delta for local_delta, update_delta in zip(local_deltas, update_deltas)]

Response: Applies the optimizer step to the set of variables.


Instruction:
Make a summary of the following Python 3 code
def computeStatsEigen(self):
        # TO-DO: figure out why this op has delays (possibly moving
        # eigenvectors around?)
        with tf.device('/cpu:0'):
            def removeNone(tensor_list):
                local_list = []
                for item in tensor_list:
                    if item is not None:
                        local_list.append(item)
                return local_list

            def copyStats(var_list):
                print("copying stats to buffer tensors before eigen decomp")
                redundant_stats = {}
                copied_list = []
                for item in var_list:
                    if item is not None:
                        if item not in redundant_stats:
                            if self._use_float64:
                                redundant_stats[item] = tf.cast(
                                    tf.identity(item), tf.float64)
                            else:
                                redundant_stats[item] = tf.identity(item)
                        copied_list.append(redundant_stats[item])
                    else:
                        copied_list.append(None)
                return copied_list
            #stats = [copyStats(self.fStats), copyStats(self.bStats)]
            #stats = [self.fStats, self.bStats]

            stats_eigen = self.stats_eigen
            computedEigen = {}
            eigen_reverse_lookup = {}
            updateOps = []
            # sync copied stats
            # with tf.control_dependencies(removeNone(stats[0]) +
            # removeNone(stats[1])):
            with tf.control_dependencies([]):
                for stats_var in stats_eigen:
                    if stats_var not in computedEigen:
                        eigens = tf.self_adjoint_eig(stats_var)
                        e = eigens[0]
                        Q = eigens[1]
                        if self._use_float64:
                            e = tf.cast(e, tf.float32)
                            Q = tf.cast(Q, tf.float32)
                        updateOps.append(e)
                        updateOps.append(Q)
                        computedEigen[stats_var] = {'e': e, 'Q': Q}
                        eigen_reverse_lookup[e] = stats_eigen[stats_var]['e']
                        eigen_reverse_lookup[Q] = stats_eigen[stats_var]['Q']

            self.eigen_reverse_lookup = eigen_reverse_lookup
            self.eigen_update_list = updateOps

        return updateOps

Response: compute the eigen decomp using copied var stats


Instruction:
How would you explain what the following Python 3 function does
def tf_step(self, time, variables, **kwargs):
        fn_loss = kwargs["fn_loss"]
        if variables is None:
            variables = tf.trainable_variables
        return tf.gradients(fn_loss, variables)

Response: Creates the TensorFlow operations for performing an optimization step on the given variables.


Instruction:
How would you implement a function in Python 3 that
applies the given step deltas to the variable values.

Response:
def apply_step(self, variables, deltas, loss_sampled):
        """
        Applies the given (and already calculated) step deltas to the variable values.

        Args:
            variables: List of variables.
            deltas: List of deltas of same length.
            loss_sampled : the sampled loss

        Returns:
            The step-applied operation. A tf.group of tf.assign_add ops.
        """
        update_stats_op = self.compute_and_apply_stats(
            loss_sampled, var_list=var_list)
        grads = [(a, b) for a, b in zip(deltas, varlist)]
        kfacOptim, _ = self.apply_gradients_kfac(grads)
        return kfacOptim


Instruction:
Write a Python 3 script to
perform an optimization step.

Response:
def minimize(self, time, variables, **kwargs):
        """
        Performs an optimization step.

        Args:
            time: Time tensor. Not used for this
            variables: List of variables to optimize.
            **kwargs:
                fn_loss : loss function tensor that is differentiated
                sampled_loss : the sampled loss from running the model.

        Returns:
            The optimization operation.
        """
        loss = kwargs["fn_loss"]
        sampled_loss = kwargs["sampled_loss"]

        min_op, _ = self.minimize_(loss, sampled_loss, var_list=variables)
        return min_op


Instruction:
How would you explain what the following Python 3 function does
def setup_components_and_tf_funcs(self, custom_getter=None):
        custom_getter = super(QDemoModel, self).setup_components_and_tf_funcs(custom_getter)

        self.demo_memory = Replay(
            states=self.states_spec,
            internals=self.internals_spec,
            actions=self.actions_spec,
            include_next_states=True,
            capacity=self.demo_memory_capacity,
            scope='demo-replay',
            summary_labels=self.summary_labels
        )

        # Import demonstration optimization.
        self.fn_import_demo_experience = tf.make_template(
            name_='import-demo-experience',
            func_=self.tf_import_demo_experience,
            custom_getter_=custom_getter
        )

        # Demonstration loss.
        self.fn_demo_loss = tf.make_template(
            name_='demo-loss',
            func_=self.tf_demo_loss,
            custom_getter_=custom_getter
        )

        # Combined loss.
        self.fn_combined_loss = tf.make_template(
            name_='combined-loss',
            func_=self.tf_combined_loss,
            custom_getter_=custom_getter
        )

        # Demonstration optimization.
        self.fn_demo_optimization = tf.make_template(
            name_='demo-optimization',
            func_=self.tf_demo_optimization,
            custom_getter_=custom_getter
        )

        return custom_getter

Response: Constructs the extra Replay memory and the demo - loss functions.


Instruction:
Create a Python 3 function to
import a single experience to memory.

Response:
def tf_import_demo_experience(self, states, internals, actions, terminal, reward):
        """
        Imports a single experience to memory.
        """
        return self.demo_memory.store(
            states=states,
            internals=internals,
            actions=actions,
            terminal=terminal,
            reward=reward
        )


Instruction:
Can you create a Python 3 function that
creates the demo loss for the states actions terminal and reward.

Response:
def tf_demo_loss(self, states, actions, terminal, reward, internals, update, reference=None):
        """
        Extends the q-model loss via the dqfd large-margin loss.
        """
        embedding = self.network.apply(x=states, internals=internals, update=update)
        deltas = list()

        for name in sorted(actions):
            action = actions[name]
            distr_params = self.distributions[name].parameterize(x=embedding)
            state_action_value = self.distributions[name].state_action_value(distr_params=distr_params, action=action)

            # Create the supervised margin loss
            # Zero for the action taken, one for all other actions, now multiply by expert margin
            if self.actions_spec[name]['type'] == 'bool':
                num_actions = 2
                action = tf.cast(x=action, dtype=util.tf_dtype('int'))
            else:
                num_actions = self.actions_spec[name]['num_actions']

            one_hot = tf.one_hot(indices=action, depth=num_actions)
            ones = tf.ones_like(tensor=one_hot, dtype=tf.float32)
            inverted_one_hot = ones - one_hot

            # max_a([Q(s,a) + l(s,a_E,a)], l(s,a_E, a) is 0 for expert action and margin value for others
            state_action_values = self.distributions[name].state_action_value(distr_params=distr_params)
            state_action_values = state_action_values + inverted_one_hot * self.expert_margin
            supervised_selector = tf.reduce_max(input_tensor=state_action_values, axis=-1)

            # J_E(Q) = max_a([Q(s,a) + l(s,a_E,a)] - Q(s,a_E)
            delta = supervised_selector - state_action_value

            action_size = util.prod(self.actions_spec[name]['shape'])
            delta = tf.reshape(tensor=delta, shape=(-1, action_size))
            deltas.append(delta)

        loss_per_instance = tf.reduce_mean(input_tensor=tf.concat(values=deltas, axis=1), axis=1)
        loss_per_instance = tf.square(x=loss_per_instance)

        return tf.reduce_mean(input_tensor=loss_per_instance, axis=0)


Instruction:
Given the following Python 3 function, write the documentation
def tf_combined_loss(self, states, internals, actions, terminal, reward, next_states, next_internals, update, reference=None):
        q_model_loss = self.fn_loss(
            states=states,
            internals=internals,
            actions=actions,
            terminal=terminal,
            reward=reward,
            next_states=next_states,
            next_internals=next_internals,
            update=update,
            reference=reference
        )

        demo_loss = self.fn_demo_loss(
            states=states,
            internals=internals,
            actions=actions,
            terminal=terminal,
            reward=reward,
            update=update,
            reference=reference
        )

        return q_model_loss + self.supervised_weight * demo_loss

Response: Combines Q - loss and demo loss.


Instruction:
How would you implement a function in Python 3 that
returns the TensorFlow variables used by the model.

Response:
def get_variables(self, include_submodules=False, include_nontrainable=False):
        """
        Returns the TensorFlow variables used by the model.

        Returns:
            List of variables.
        """
        model_variables = super(QDemoModel, self).get_variables(
            include_submodules=include_submodules,
            include_nontrainable=include_nontrainable
        )

        if include_nontrainable:
            demo_memory_variables = self.demo_memory.get_variables()
            model_variables += demo_memory_variables

        return model_variables


Instruction:
Create a Python 3 function for
storing demonstrations in the demo memory.

Response:
def import_demo_experience(self, states, internals, actions, terminal, reward):
        """
        Stores demonstrations in the demo memory.
        """
        fetches = self.import_demo_experience_output

        feed_dict = self.get_feed_dict(
            states=states,
            internals=internals,
            actions=actions,
            terminal=terminal,
            reward=reward
        )

        self.monitored_session.run(fetches=fetches, feed_dict=feed_dict)


Instruction:
Can you write a function in Python 3 where it
performs a demo update by calling the demo optimization operation.

Response:
def demo_update(self):
        """
        Performs a demonstration update by calling the demo optimization operation.
        Note that the batch data does not have to be fetched from the demo memory as this is now part of
        the TensorFlow operation of the demo update.
        """
        fetches = self.demo_optimization_output

        self.monitored_session.run(fetches=fetches)


Instruction:
Here you have a function in Python 3, explain what it does
def from_config(config, kwargs=None):
        return util.get_object(
            obj=config,
            predefined=tensorforce.core.optimizers.solvers.solvers,
            kwargs=kwargs
        )

Response: Creates a solver from a specification dict.


Instruction:
Can you generate the documentation for the following Python 3 function
def tf_step(self, time, variables, **kwargs):
        arguments = kwargs["arguments"]
        fn_loss = kwargs["fn_loss"]
        loss = fn_loss(**arguments)

        # Force loss value to be calculated.
        with tf.control_dependencies(control_inputs=(loss,)):
            # Trivial operation to enforce control dependency
            previous_variables = [variable + 0.0 for variable in variables]

        # The actual tensorflow minimize op.
        with tf.control_dependencies(control_inputs=previous_variables):
            applied = self.tf_optimizer.minimize(loss=loss, var_list=variables)  # colocate_gradients_with_ops=True

        # Return deltas after actually having change the variables.
        with tf.control_dependencies(control_inputs=(applied,)):
            return [
                variable - previous_variable
                for variable, previous_variable in zip(variables, previous_variables)
            ]

Response: This function is called by the tensorflow optimizer to perform a single step of the tensorflow optimizer on the current model.