Instruction:
Here you have a function in Python 3, explain what it does
def build_modules(is_training, vocab_size):

  # Construct the custom getter which implements Bayes by Backprop.
  if is_training:
    estimator_mode = tf.constant(bbb.EstimatorModes.sample)
  else:
    estimator_mode = tf.constant(bbb.EstimatorModes.mean)
  lstm_bbb_custom_getter = bbb.bayes_by_backprop_getter(
      posterior_builder=lstm_posterior_builder,
      prior_builder=custom_scale_mixture_prior_builder,
      kl_builder=bbb.stochastic_kl_builder,
      sampling_mode_tensor=estimator_mode)
  non_lstm_bbb_custom_getter = bbb.bayes_by_backprop_getter(
      posterior_builder=non_lstm_posterior_builder,
      prior_builder=custom_scale_mixture_prior_builder,
      kl_builder=bbb.stochastic_kl_builder,
      sampling_mode_tensor=estimator_mode)

  embed_layer = snt.Embed(
      vocab_size=vocab_size,
      embed_dim=FLAGS.embedding_size,
      custom_getter=non_lstm_bbb_custom_getter,
      name="input_embedding")

  cores = []
  for i in range(FLAGS.n_layers):
    cores.append(
        snt.LSTM(FLAGS.hidden_size,
                 custom_getter=lstm_bbb_custom_getter,
                 forget_bias=0.0,
                 name="lstm_layer_{}".format(i)))

  rnn_core = snt.DeepRNN(
      cores,
      skip_connections=False,
      name="deep_lstm_core")

  # Do BBB on weights but not biases of output layer.
  output_linear = snt.Linear(
      vocab_size, custom_getter={"w": non_lstm_bbb_custom_getter})
  return embed_layer, rnn_core, output_linear

Response: Construct the modules used in the graph.


Instruction:
Write a Python 3 script for
computing the log loss given predictions and targets.

Response:
def build_loss(model_logits, sparse_targets):
  """Compute the log loss given predictions and targets."""
  time_major_shape = [FLAGS.unroll_steps, FLAGS.batch_size]
  flat_batch_shape = [FLAGS.unroll_steps * FLAGS.batch_size, -1]
  xent = tf.nn.sparse_softmax_cross_entropy_with_logits(
      logits=tf.reshape(model_logits, flat_batch_shape),
      labels=tf.reshape(sparse_targets, flat_batch_shape[:-1]))
  xent = tf.reshape(xent, time_major_shape)
  # Sum over the sequence.
  sequence_neg_log_prob = tf.reduce_sum(xent, axis=0)
  # Average over the batch.
  return tf.reduce_mean(sequence_neg_log_prob, axis=0)


Instruction:
Can you implement a function in Python 3 that
runs a PTB training set checkpointing the weights.

Response:
def train(logdir):
  """Run a network on the PTB training set, checkpointing the weights."""

  ptb_train = PTB(
      name="ptb_train",
      subset="train",
      seq_len=FLAGS.unroll_steps,
      batch_size=FLAGS.batch_size)

  # Connect to training set.
  data_ops = ptb_train()
  embed_layer, rnn_core, output_linear = build_modules(
      is_training=True, vocab_size=ptb_train.vocab_size)
  prediction_logits, zero_state_op = build_logits(
      data_ops, embed_layer, rnn_core, output_linear, name_prefix="train")
  data_loss = build_loss(prediction_logits, data_ops.sparse_target)

  # Add the KL cost.
  total_kl_cost = bbb.get_total_kl_cost()
  num_dataset_elements = FLAGS.batch_size * ptb_train.num_batches
  scaled_kl_cost = total_kl_cost / num_dataset_elements
  total_loss = tf.add(scaled_kl_cost, data_loss)

  # Optimize as usual.
  global_step = tf.get_variable(
      "num_weight_updates",
      initializer=tf.constant(0, dtype=tf.int32, shape=()),
      collections=[tf.GraphKeys.GLOBAL_VARIABLES, tf.GraphKeys.GLOBAL_STEP])

  learning_rate = tf.get_variable(
      "lr", initializer=tf.constant(FLAGS.lr_start, shape=(), dtype=tf.float32))
  learning_rate_update = learning_rate.assign(learning_rate * FLAGS.lr_decay)
  optimizer = tf.train.GradientDescentOptimizer(
      learning_rate=learning_rate)
  optimizer = GlobalNormClippingOptimizer(optimizer, clip_norm=5.0)

  with tf.control_dependencies([optimizer.minimize(total_loss)]):
    global_step_and_train = global_step.assign_add(1)

  # Connect to valid set.
  ptb_valid = PTB(
      name="ptb_valid",
      subset="valid",
      seq_len=FLAGS.unroll_steps,
      batch_size=FLAGS.batch_size)
  valid_data_ops = ptb_valid()
  valid_logits, zero_valid_state = build_logits(
      valid_data_ops, embed_layer, rnn_core, output_linear, name_prefix="valid")
  valid_loss = build_loss(valid_logits, valid_data_ops.sparse_target)

  # Compute metrics for the sake of monitoring training.
  predictions = tf.cast(
      tf.argmax(prediction_logits, axis=-1), tf.int32, name="pred")
  correct_prediction_mask = tf.cast(
      tf.equal(predictions, data_ops.sparse_target), tf.int32)
  accuracy = tf.reduce_mean(
      tf.cast(correct_prediction_mask, tf.float32), name="acc")
  error_rate = tf.subtract(1.0, accuracy, name="err")
  label_probs = tf.nn.softmax(prediction_logits, dim=-1)
  predictive_entropy = tf.reduce_mean(
      label_probs * tf.log(label_probs + 1e-12) * -1.0)

  # Create tf.summary ops.
  log_ops_to_run = {
      "scalar": collections.OrderedDict([
          ("task_loss", data_loss),
          ("train_err_rate", error_rate),
          ("pred_entropy", predictive_entropy),
          ("learning_rate", learning_rate),
          ("elbo_loss", total_loss),
          ("kl_cost", total_kl_cost),
          ("scaled_kl_cost", scaled_kl_cost),
      ]),
      "text": collections.OrderedDict([
          ("labels", ptb_train.to_string_tensor(data_ops.sparse_target)),
          ("predictions", ptb_train.to_string_tensor(predictions))
      ])
  }

  for name, tensor in log_ops_to_run["scalar"].items():
    tf.summary.scalar(os.path.join("train", name), tensor)

  # The remaining logic runs the training loop and logging.
  summary_writer = tf.summary.FileWriterCache.get(logdir=logdir)
  tf.logging.info(
      "Beginning training for {} epochs, each with {} batches.".format(
          FLAGS.num_training_epochs, ptb_train.num_batches))
  with tf.train.MonitoredTrainingSession(
      is_chief=True, checkpoint_dir=logdir, save_summaries_secs=10) as sess:
    num_updates_v = _run_session_with_no_hooks(sess, global_step)
    epoch_idx_start, step_idx_start = divmod(
        num_updates_v, ptb_train.num_batches)
    tf.logging.info("On start, epoch: {}\t step: {}".format(
        epoch_idx_start, step_idx_start))
    for epoch_idx in range(epoch_idx_start, FLAGS.num_training_epochs):
      tf.logging.info("Beginning Epoch {}/{}".format(
          epoch_idx, FLAGS.num_training_epochs))
      tf.logging.info(
          ("Beginning by evaluating on the validation set, which has "
           "{} batches.".format(ptb_valid.num_batches)))
      valid_cost = 0
      valid_steps = 0
      _run_session_with_no_hooks(sess, zero_valid_state)
      for _ in range(ptb_valid.num_batches):
        valid_cost_v, num_updates_v = _run_session_with_no_hooks(
            sess, [valid_loss, global_step])
        valid_cost += valid_cost_v
        valid_steps += FLAGS.unroll_steps
      tf.logging.info("Validation set perplexity: {}".format(
          np.exp(valid_cost / valid_steps)))
      summary = tf.summary.Summary()
      summary.value.add(
          tag="valid/word_level_perplexity",
          simple_value=np.exp(valid_cost / valid_steps))
      summary_writer.add_summary(summary, num_updates_v)

      # Run a training epoch.
      epoch_cost = 0
      epoch_steps = 0
      for batch_idx in range(step_idx_start, ptb_train.num_batches):
        scalars_res, num_updates_v = sess.run(
            [log_ops_to_run["scalar"], global_step_and_train])
        epoch_cost += scalars_res["task_loss"]
        epoch_steps += FLAGS.unroll_steps
        if (batch_idx - 1) % FLAGS.print_every_batches == 0:
          summary = tf.summary.Summary()
          summary.value.add(
              tag="train/word_level_perplexity",
              simple_value=np.exp(epoch_cost / epoch_steps))
          summary_writer.add_summary(summary, num_updates_v)
          scalars_res, strings_res = _run_session_with_no_hooks(
              sess, [log_ops_to_run["scalar"], log_ops_to_run["text"]])
          tf.logging.info("Num weight updates: {}".format(num_updates_v))
          for name, result in scalars_res.items():
            tf.logging.info("{}: {}".format(name, result))
          for name, result in strings_res.items():
            tf.logging.info("{}: {}".format(name, result))

      word_level_perplexity = np.exp(epoch_cost / epoch_steps)
      tf.logging.info(
          "Train Perplexity after Epoch {}: {}".format(
              epoch_idx, word_level_perplexity))

      end_of_epoch_fetches = [zero_state_op]
      if epoch_idx >= FLAGS.high_lr_epochs:
        end_of_epoch_fetches.append(learning_rate_update)
      _run_session_with_no_hooks(sess, end_of_epoch_fetches)

  tf.logging.info("Done training. Thanks for your time.")


Instruction:
How would you explain what the following Python 3 function does
def _ensure_dependency_available_at_version(package_name, min_version):

  try:
    pkg = importlib.import_module(package_name)
  except ImportError:
    pip_name = package_name.replace('_', '-')
    raise SystemError(
        'Sonnet requires %s (minimum version %s) to be installed. '
        'If using pip, run `pip install %s` or '
        '`pip install %s-gpu`' % (
            package_name, min_version, pip_name, pip_name))

  installed_version = semantic_version.Version(pkg.__version__)
  version_spec = semantic_version.Spec('>=' + min_version)
  if not version_spec.match(installed_version):
    raise SystemError(
        '%s version %s is installed, but Sonnet requires at least version %s.' %
        (package_name, pkg.__version__, min_version))

Response: Throw helpful error if required dependencies not available at the given version.


Instruction:
Implement a function in Python 3 to
create an op that scales gradients using a Defun.

Response:
def _scale_gradient_op(dtype):
  """Create an op that scales gradients using a Defun.

  The tensorflow Defun decorator creates an op and tensorflow caches these ops
  automatically according to `func_name`. Using a Defun decorator twice with the
  same `func_name` does not create a new op, instead the cached op is used.

  This method produces a new op the first time it is called with a given `dtype`
  argument, and then uses the cached op each time it is called after that with
  the same `dtype`. The scale value is given as an argument for the forward pass
  method so that it can be used in the backwards pass.

  Args:
    dtype: the dtype of the net whose gradient is being scaled.

  Returns:
    The op that scales gradients.
  """

  def scale_gradient_backward(op, grad):
    scale = op.inputs[1]
    scaled_grad = grad * scale
    return scaled_grad, None

  # Note that if the forward pass implementation involved the creation of ops,
  # _scale_gradient_op would require some memoization mechanism.
  def scale_gradient_forward(x, scale):
    del scale  # Unused.
    return x

  func_name = "ScaleGradient_{}".format(dtype.name)
  return function.Defun(
      dtype, dtype,
      python_grad_func=scale_gradient_backward,
      func_name=func_name)(scale_gradient_forward)


Instruction:
How would you implement a function in Python 3 that
scales gradients on the backwards pass.

Response:
def scale_gradient(net, scale, name="scale_gradient"):
  """Scales gradients for the backwards pass.

  This might be used to, for example, allow one part of a model to learn at a
  lower rate than the rest.

  WARNING: Think carefully about how your optimizer works. If, for example, you
  use rmsprop, the gradient is always rescaled (with some additional epsilon)
  towards unity. This means `scale_gradient` won't have the effect of
  lowering the learning rate.

  If `scale` is `0.0`, this op reduces to `tf.stop_gradient`. If `scale`
  is `1.0`, this op reduces to `tf.identity`.

  Args:
    net: A `tf.Tensor` or in eager mode a callable that produces a `tf.Tensor`.
    scale: The scale factor for the gradient on the backwards pass.
    name: A name for the operation (optional).

  Returns:
    In graph mode returns a `tf.Tensor` with the same type as the input tensor.
    In eager mode returns a callable wrapping `net` whose gradients are scaled.

  Raises:
    ValueError: If `net` dtype is non-float and `scale` is not zero or one.
  """
  if tf.executing_eagerly():
    if not callable(net):
      raise ValueError(
          "In eager mode `net` must be a callable (similar to how optimizers "
          "must be used when executing eagerly).")
    return tfe.defun(lambda *a, **k: scale_gradient(net(*a, **k), scale, name))

  if scale == 0.0:
    return tf.stop_gradient(net, name=name)
  elif scale == 1.0:
    return tf.identity(net, name=name)
  else:
    if not net.dtype.is_floating:
      raise ValueError("clip_gradient does not support non-float `net` inputs.")

    with tf.name_scope(name, "scale_gradient", values=[net]):
      dtype = net.dtype.base_dtype  # Convert ref dtypes to regular dtypes.
      scale_tensor = tf.convert_to_tensor(scale, dtype=dtype)

      scale_gradient_op = _scale_gradient_op(dtype)
      output = scale_gradient_op(net, scale_tensor)
      output.set_shape(net.get_shape())

    return output


Instruction:
Can you generate a brief explanation for the following Python 3 code
def build_and_train(iterations, log_stride, test=False):

  # Test mode settings.
  batch_size = 2 if test else FLAGS.batch_size
  num_mems = 2 if test else FLAGS.num_mems
  num_heads = 1 if test else FLAGS.num_mems
  num_blocks = 1 if test else FLAGS.num_mems
  head_size = 4 if test else FLAGS.head_size
  max_length = 3 if test else FLAGS.max_length
  max_nest = 2 if test else FLAGS.max_nest
  mlp_size = (20,) if test else (256, 256, 256, 256)

  with tf.Graph().as_default():
    t0 = time.time()

    # Initialize the dataset.
    lte_train = learn_to_execute.LearnToExecute(
        batch_size, max_length, max_nest)
    lte_test = learn_to_execute.LearnToExecute(
        batch_size, max_length, max_nest, mode=learn_to_execute.Mode.TEST)
    train_data_iter = lte_train.make_one_shot_iterator().get_next()
    test_data_iter = lte_test.make_one_shot_iterator().get_next()
    output_size = lte_train.state.vocab_size

    # Create the model.
    core = snt.RelationalMemory(
        mem_slots=num_mems,
        head_size=head_size,
        num_heads=num_heads,
        num_blocks=num_blocks,
        gate_style=FLAGS.gate_style)
    final_mlp = snt.nets.MLP(
        output_sizes=mlp_size,
        activate_final=True)
    model = SequenceModel(
        core=core,
        target_size=output_size,
        final_mlp=final_mlp)
    tf.logging.info("Instantiated models ({:3f})".format(time.time() - t0))

    # Define the loss & accuracy.
    def loss_fn(inputs, targets, input_sequence_length, output_sequence_length):
      """Creates the loss and the exports."""
      logits = model(
          inputs, targets, input_sequence_length, output_sequence_length)
      targets = tf.cast(targets, tf.int32)
      sq_sz_out_max = targets.shape[0].value

      # Create a mask to ignore accuracy on buffer characters.
      sequence_sizes = tf.cast(output_sequence_length, tf.float32)
      lengths_transposed = tf.expand_dims(sequence_sizes, 1)
      range_row = tf.expand_dims(
          tf.range(0, sq_sz_out_max, 1, dtype=tf.float32), 0)
      mask = tf.cast(tf.transpose(tf.less(range_row, lengths_transposed)),
                     tf.float32)

      # Compute token accuracy and solved.
      correct = tf.equal(tf.argmax(logits, 2), tf.argmax(targets, 2))
      solved = tf.reduce_all(tf.boolean_mask(correct, tf.squeeze(mask)), axis=0)
      token_acc = tf.reduce_sum(tf.cast(correct, tf.float32) * mask)
      token_acc /= tf.reduce_sum(sequence_sizes)

      # Compute Loss.
      mask = tf.cast(tf.tile(tf.expand_dims(mask, 2), (1, 1, logits.shape[2])),
                     tf.float32)
      masked_logits = logits * mask
      masked_target = tf.cast(targets, tf.float32) * mask
      logits_flat = tf.reshape(masked_logits,
                               [sq_sz_out_max * batch_size, -1])
      target_flat = tf.reshape(masked_target,
                               [sq_sz_out_max * batch_size, -1])
      xent = tf.nn.softmax_cross_entropy_with_logits(logits=logits_flat,
                                                     labels=target_flat)
      loss = tf.reduce_mean(xent)
      return loss, token_acc, solved

    # Get training step counter.
    global_step = tf.train.get_or_create_global_step()

    # Create the optimizer.
    learning_rate_op = tf.reduce_max([
        tf.train.exponential_decay(
            FLAGS.learning_rate,
            global_step,
            decay_steps=FLAGS.epochs // 100,
            decay_rate=0.9,
            staircase=False),
        FLAGS.min_learning_rate
    ])
    optimizer = tf.train.AdamOptimizer(learning_rate_op)

    # Compute loss, accuracy & the step op.
    inputs, targets, _, input_lengths, output_lengths = train_data_iter
    train_loss, train_acc, train_sol = loss_fn(
        inputs, targets, input_lengths, output_lengths)
    step_op = optimizer.minimize(train_loss, global_step=global_step)
    inputs, targets, _, input_lengths, output_lengths = test_data_iter
    _, test_acc, test_sol = loss_fn(
        inputs, targets, input_lengths, output_lengths)
    tf.logging.info("Created losses and optimizers ({:3f})".format(
        time.time() - t0))

    # Begin Training.
    t0 = time.time()
    tf.logging.info("Starting training ({:3f})".format(time.time() - t0))
    with tf.train.SingularMonitoredSession() as sess:
      for it in six.moves.range(iterations):
        sess.run([step_op, learning_rate_op])
        if it % log_stride == 0:
          loss_v, train_acc_v, test_acc_v, train_sol_v, test_sol_v = sess.run([
              train_loss, train_acc, test_acc, train_sol, test_sol])
          elapsed = time.time() - t0
          tf.logging.info(
              "iter: {:2d}, train loss {:3f}; train acc {:3f}; test acc {:3f};"
              " train solved {:3f}; test solved {:3f}; ({:3f})".format(
                  it, loss_v, train_acc_v, test_acc_v, train_sol_v, test_sol_v,
                  elapsed))

Response: Construct the data model loss and optimizer then train.


Instruction:
Implement a function in Python 3 to
build the seq2seq model.

Response:
def _build(
      self, inputs, targets, input_sequence_length, output_sequence_length):
    """Dynamic unroll across input objects.

    Args:
      inputs: tensor (input_sequence_length x batch x feature_size). Encoder
          sequence.
      targets: tensor (output_sequence_length x batch x feature_size). Decoder
          sequence.
      input_sequence_length: tensor (batch). Size of each batched input
          sequence.
      output_sequence_length: tensor (batch). Size of each batched target
          sequence.

    Returns:
      Tensor (batch x num_objects); logits indicating the reference objects.
    """
    # Connect decoding steps.
    batch_size = inputs.get_shape()[1]
    initial_state = self._core.initial_state(batch_size, trainable=False)
    _, state = tf.nn.dynamic_rnn(
        cell=self._core,
        inputs=inputs,
        sequence_length=input_sequence_length,
        time_major=True,
        initial_state=initial_state
    )
    # Connect decoding steps.
    zero_input = tf.zeros(shape=targets.get_shape())
    output_sequence, _ = tf.nn.dynamic_rnn(
        cell=self._core,
        inputs=zero_input,  # Non-autoregressive model.  Zeroed input.
        sequence_length=output_sequence_length,
        initial_state=state,
        time_major=True)
    outputs = snt.BatchApply(self._final_mlp)(output_sequence)
    logits = snt.BatchApply(snt.Linear(self._target_size))(outputs)
    tf.logging.info("Connected seq2seq model.")
    return logits


Instruction:
Explain what the following Python 3 code does
def lstm_with_recurrent_dropout(hidden_size, keep_prob=0.5, **kwargs):

  lstm = LSTM(hidden_size, **kwargs)
  return RecurrentDropoutWrapper(lstm, LSTMState(keep_prob, None)), lstm

Response: Returns a new LSTM with recurrent dropout enabled.


Instruction:
Write a Python 3 script for
returning a new LSTM with recurrent dropout enabled for training and test LSTM.

Response:
def lstm_with_zoneout(hidden_size, keep_prob_c=0.5, keep_prob_h=0.95, **kwargs):
  """LSTM with recurrent dropout.

  Args:
    hidden_size: the LSTM hidden size.
    keep_prob_c: the probability to use the new value of the cell state rather
      than freezing it.
    keep_prob_h: the probability to use the new value of the hidden state
      rather than freezing it.
    **kwargs: Extra keyword arguments to pass to the LSTM.

  Returns:
    A tuple (train_lstm, test_lstm) where train_lstm is an LSTM with
    recurrent dropout enabled to be used for training and test_lstm
    is the same LSTM without zoneout.
  """

  lstm = LSTM(hidden_size, **kwargs)
  keep_probs = LSTMState(keep_prob_h, keep_prob_c)
  train_lstm = ZoneoutWrapper(lstm, keep_probs, is_training=True)
  test_lstm = ZoneoutWrapper(lstm, keep_probs, is_training=False)
  return train_lstm, test_lstm


Instruction:
Here you have a function in Python 3, explain what it does
def highway_core_with_recurrent_dropout(
    hidden_size,
    num_layers,
    keep_prob=0.5,
    **kwargs):
  """Highway core with recurrent dropout.

  Args:
    hidden_size: (int) Hidden size dimensionality.
    num_layers: (int) Number of highway layers.
    keep_prob: the probability to keep an entry when applying dropout.
    **kwargs: Extra keyword arguments to pass to the highway core.

  Returns:
    A tuple (train_core, test_core) where train_core is a higway core with
    recurrent dropout enabled to be used for training and test_core is the
    same highway core without recurrent dropout.
  """

  core = HighwayCore(hidden_size, num_layers, **kwargs)
  return RecurrentDropoutWrapper(core, keep_prob), core

Response: Highway core with recurrent dropout.


Instruction:
Write a Python 3 script to
return the set of possible variable initializers that may be used for the given class.

Response:
def get_possible_initializer_keys(cls, use_peepholes=False,
                                    use_projection=False):
    """Returns the keys the dictionary of variable initializers may contain.

    The set of all possible initializer keys are:
      w_gates:  weight for gates
      b_gates:  bias of gates
      w_f_diag: weight for prev_cell -> forget gate peephole
      w_i_diag: weight for prev_cell -> input gate peephole
      w_o_diag: weight for prev_cell -> output gate peephole

    Args:
      cls:The class.
      use_peepholes: Boolean that indicates whether peephole connections are
        used.
      use_projection: Boolean that indicates whether a recurrent projection
        layer is used.

    Returns:
      Set with strings corresponding to the strings that may be passed to the
        constructor.
    """

    possible_keys = cls.POSSIBLE_INITIALIZER_KEYS.copy()
    if not use_peepholes:
      possible_keys.difference_update(
          {cls.W_F_DIAG, cls.W_I_DIAG, cls.W_O_DIAG})
    if not use_projection:
      possible_keys.difference_update({cls.W_H_PROJECTION})
    return possible_keys


Instruction:
Can you tell what is the following Python 3 function doing
def _build(self, inputs, prev_state):
    prev_hidden, prev_cell = prev_state

    # pylint: disable=invalid-unary-operand-type
    if self._hidden_clip_value is not None:
      prev_hidden = tf.clip_by_value(
          prev_hidden, -self._hidden_clip_value, self._hidden_clip_value)
    if self._cell_clip_value is not None:
      prev_cell = tf.clip_by_value(
          prev_cell, -self._cell_clip_value, self._cell_clip_value)
    # pylint: enable=invalid-unary-operand-type

    self._create_gate_variables(inputs.get_shape(), inputs.dtype)

    # pylint false positive: calling module of same file;
    # pylint: disable=not-callable

    # Parameters of gates are concatenated into one multiply for efficiency.
    inputs_and_hidden = tf.concat([inputs, prev_hidden], 1)
    gates = tf.matmul(inputs_and_hidden, self._w_xh)

    if self._use_layer_norm:
      gates = layer_norm.LayerNorm()(gates)

    gates += self._b

    # i = input_gate, j = next_input, f = forget_gate, o = output_gate
    i, j, f, o = tf.split(value=gates, num_or_size_splits=4, axis=1)

    if self._use_peepholes:  # diagonal connections
      self._create_peephole_variables(inputs.dtype)
      f += self._w_f_diag * prev_cell
      i += self._w_i_diag * prev_cell

    forget_mask = tf.sigmoid(f + self._forget_bias)
    next_cell = forget_mask * prev_cell + tf.sigmoid(i) * tf.tanh(j)
    cell_output = next_cell
    if self._use_peepholes:
      cell_output += self._w_o_diag * cell_output
    next_hidden = tf.tanh(cell_output) * tf.sigmoid(o)

    if self._use_projection:
      next_hidden = tf.matmul(next_hidden, self._w_h_projection)

    return next_hidden, LSTMState(hidden=next_hidden, cell=next_cell)

Response: Connects the LSTM module into the graph.


Instruction:
How would you explain what the following Python 3 function does
def _create_gate_variables(self, input_shape, dtype):
    if len(input_shape) != 2:
      raise ValueError(
          "Rank of shape must be {} not: {}".format(2, len(input_shape)))

    equiv_input_size = self._hidden_state_size + input_shape.dims[1].value
    initializer = basic.create_linear_initializer(equiv_input_size)

    self._w_xh = tf.get_variable(
        self.W_GATES,
        shape=[equiv_input_size, 4 * self._hidden_size],
        dtype=dtype,
        initializer=self._initializers.get(self.W_GATES, initializer),
        partitioner=self._partitioners.get(self.W_GATES),
        regularizer=self._regularizers.get(self.W_GATES))
    self._b = tf.get_variable(
        self.B_GATES,
        shape=[4 * self._hidden_size],
        dtype=dtype,
        initializer=self._initializers.get(self.B_GATES, initializer),
        partitioner=self._partitioners.get(self.B_GATES),
        regularizer=self._regularizers.get(self.B_GATES))
    if self._use_projection:
      w_h_initializer = basic.create_linear_initializer(self._hidden_size)
      self._w_h_projection = tf.get_variable(
          self.W_H_PROJECTION,
          shape=[self._hidden_size, self._hidden_state_size],
          dtype=dtype,
          initializer=self._initializers.get(self.W_H_PROJECTION,
                                             w_h_initializer),
          partitioner=self._partitioners.get(self.W_H_PROJECTION),
          regularizer=self._regularizers.get(self.W_H_PROJECTION))

Response: Initialize the variables used for the gates.


Instruction:
Here you have a function in Python 3, explain what it does
def _create_peephole_variables(self, dtype):
    self._w_f_diag = tf.get_variable(
        self.W_F_DIAG,
        shape=[self._hidden_size],
        dtype=dtype,
        initializer=self._initializers.get(self.W_F_DIAG),
        partitioner=self._partitioners.get(self.W_F_DIAG),
        regularizer=self._regularizers.get(self.W_F_DIAG))
    self._w_i_diag = tf.get_variable(
        self.W_I_DIAG,
        shape=[self._hidden_size],
        dtype=dtype,
        initializer=self._initializers.get(self.W_I_DIAG),
        partitioner=self._partitioners.get(self.W_I_DIAG),
        regularizer=self._regularizers.get(self.W_I_DIAG))
    self._w_o_diag = tf.get_variable(
        self.W_O_DIAG,
        shape=[self._hidden_size],
        dtype=dtype,
        initializer=self._initializers.get(self.W_O_DIAG),
        partitioner=self._partitioners.get(self.W_O_DIAG),
        regularizer=self._regularizers.get(self.W_O_DIAG))

Response: Initialize the variables used for the peephole connections.


Instruction:
Implement a function in Python 3 to
build the default start state tensor of zeros.

Response:
def initial_state(self, batch_size, dtype=tf.float32, trainable=False,
                    trainable_initializers=None, trainable_regularizers=None,
                    name=None):
    """Builds the default start state tensor of zeros."""
    core_initial_state = self._core.initial_state(
        batch_size, dtype=dtype, trainable=trainable,
        trainable_initializers=trainable_initializers,
        trainable_regularizers=trainable_regularizers, name=name)

    dropout_masks = [None] * len(self._dropout_state_size)
    def set_dropout_mask(index, state, keep_prob):
      if index is not None:
        ones = tf.ones_like(state, dtype=dtype)
        dropout_masks[index] = tf.nn.dropout(ones, keep_prob=keep_prob)
    tf.contrib.framework.nest.map_structure(
        set_dropout_mask,
        self._dropout_indexes, core_initial_state, self._keep_probs)

    return core_initial_state, dropout_masks


Instruction:
How would you explain what the following Python 3 function does
def initial_state(self, batch_size, dtype=tf.float32, trainable=False,
                    trainable_initializers=None, trainable_regularizers=None,
                    name=None):
    """Builds the default start state tensor of zeros."""
    return self._core.initial_state(
        batch_size, dtype=dtype, trainable=trainable,
        trainable_initializers=trainable_initializers,
        trainable_regularizers=trainable_regularizers, name=name)

Response: Builds the default start state tensor of zeros.


Instruction:
Implement a Python 3 function for
wrapping this RNNCore with the additional input to the BatchNorm LSTM.

Response:
def with_batch_norm_control(self, is_training, test_local_stats=True):
    """Wraps this RNNCore with the additional control input to the `BatchNorm`s.

    Example usage:

      lstm = snt.BatchNormLSTM(4)
      is_training = tf.placeholder(tf.bool)
      rnn_input = ...
      my_rnn = rnn.rnn(lstm.with_batch_norm_control(is_training), rnn_input)

    Args:
      is_training: Boolean that indicates whether we are in
        training mode or testing mode. When in training mode, the batch norm
        statistics are taken from the given batch, and moving statistics are
        updated. When in testing mode, the moving statistics are not updated,
        and in addition if `test_local_stats` is False then the moving
        statistics are used for the batch statistics. See the `BatchNorm` module
        for more details.
      test_local_stats: Boolean scalar indicated whether to use local
        batch statistics in test mode.

    Returns:
      snt.RNNCore wrapping this class with the extra input(s) added.
    """
    return BatchNormLSTM.CoreWithExtraBuildArgs(
        self, is_training=is_training, test_local_stats=test_local_stats)


Instruction:
Can you generate the documentation for the following Python 3 function
def get_possible_initializer_keys(
      cls, use_peepholes=False, use_batch_norm_h=True, use_batch_norm_x=False,
      use_batch_norm_c=False):
    """Returns the keys the dictionary of variable initializers may contain.

    The set of all possible initializer keys are:
      w_gates:  weight for gates
      b_gates:  bias of gates
      w_f_diag: weight for prev_cell -> forget gate peephole
      w_i_diag: weight for prev_cell -> input gate peephole
      w_o_diag: weight for prev_cell -> output gate peephole
      gamma_h:  batch norm scaling for previous_hidden -> gates
      gamma_x:  batch norm scaling for input -> gates
      gamma_c:  batch norm scaling for cell -> output
      beta_c:   batch norm bias for cell -> output

    Args:
      cls:The class.
      use_peepholes: Boolean that indicates whether peephole connections are
        used.
      use_batch_norm_h: Boolean that indicates whether to apply batch
        normalization at the previous_hidden -> gates contribution. If you are
        experimenting with batch norm then this may be the most effective to
        turn on.
      use_batch_norm_x: Boolean that indicates whether to apply batch
        normalization at the input -> gates contribution.
      use_batch_norm_c: Boolean that indicates whether to apply batch
        normalization at the cell -> output contribution.

    Returns:
      Set with strings corresponding to the strings that may be passed to the
        constructor.
    """

    possible_keys = cls.POSSIBLE_INITIALIZER_KEYS.copy()
    if not use_peepholes:
      possible_keys.difference_update(
          {cls.W_F_DIAG, cls.W_I_DIAG, cls.W_O_DIAG})
    if not use_batch_norm_h:
      possible_keys.remove(cls.GAMMA_H)
    if not use_batch_norm_x:
      possible_keys.remove(cls.GAMMA_X)
    if not use_batch_norm_c:
      possible_keys.difference_update({cls.GAMMA_C, cls.BETA_C})
    return possible_keys

Response: Returns the set of possible initialization keys for the given class.


Instruction:
Given the following Python 3 function, write the documentation
def _build(self, inputs, prev_state, is_training=None, test_local_stats=True):
    if is_training is None:
      raise ValueError("Boolean is_training flag must be explicitly specified "
                       "when using batch normalization.")

    if self._max_unique_stats == 1:
      prev_hidden, prev_cell = prev_state
      time_step = None
    else:
      prev_hidden, prev_cell, time_step = prev_state

    # pylint: disable=invalid-unary-operand-type
    if self._hidden_clip_value is not None:
      prev_hidden = tf.clip_by_value(
          prev_hidden, -self._hidden_clip_value, self._hidden_clip_value)
    if self._cell_clip_value is not None:
      prev_cell = tf.clip_by_value(
          prev_cell, -self._cell_clip_value, self._cell_clip_value)
    # pylint: enable=invalid-unary-operand-type

    self._create_gate_variables(inputs.get_shape(), inputs.dtype)
    self._create_batch_norm_variables(inputs.dtype)

    # pylint false positive: calling module of same file;
    # pylint: disable=not-callable

    if self._use_batch_norm_h or self._use_batch_norm_x:
      gates_h = tf.matmul(prev_hidden, self._w_h)
      gates_x = tf.matmul(inputs, self._w_x)
      if self._use_batch_norm_h:
        gates_h = self._gamma_h * self._batch_norm_h(gates_h,
                                                     time_step,
                                                     is_training,
                                                     test_local_stats)
      if self._use_batch_norm_x:
        gates_x = self._gamma_x * self._batch_norm_x(gates_x,
                                                     time_step,
                                                     is_training,
                                                     test_local_stats)
      gates = gates_h + gates_x
    else:
      # Parameters of gates are concatenated into one multiply for efficiency.
      inputs_and_hidden = tf.concat([inputs, prev_hidden], 1)
      gates = tf.matmul(inputs_and_hidden, self._w_xh)

    gates += self._b

    # i = input_gate, j = next_input, f = forget_gate, o = output_gate
    i, j, f, o = tf.split(value=gates, num_or_size_splits=4, axis=1)

    if self._use_peepholes:  # diagonal connections
      self._create_peephole_variables(inputs.dtype)
      f += self._w_f_diag * prev_cell
      i += self._w_i_diag * prev_cell

    forget_mask = tf.sigmoid(f + self._forget_bias)
    next_cell = forget_mask * prev_cell + tf.sigmoid(i) * tf.tanh(j)
    cell_output = next_cell
    if self._use_batch_norm_c:
      cell_output = (self._beta_c
                     + self._gamma_c * self._batch_norm_c(cell_output,
                                                          time_step,
                                                          is_training,
                                                          test_local_stats))
    if self._use_peepholes:
      cell_output += self._w_o_diag * cell_output
    next_hidden = tf.tanh(cell_output) * tf.sigmoid(o)

    if self._max_unique_stats == 1:
      return next_hidden, (next_hidden, next_cell)
    else:
      return next_hidden, (next_hidden, next_cell, time_step + 1)

Response: Connects the LSTM module into the graph.


Instruction:
Can you tell what is the following Python 3 function doing
def _create_batch_norm_variables(self, dtype):
    # The paper recommends a value of 0.1 for good gradient flow through the
    # tanh nonlinearity (although doesn't say whether this is for all gammas,
    # or just some).
    gamma_initializer = tf.constant_initializer(0.1)

    if self._use_batch_norm_h:
      self._gamma_h = tf.get_variable(
          self.GAMMA_H,
          shape=[4 * self._hidden_size],
          dtype=dtype,
          initializer=self._initializers.get(self.GAMMA_H, gamma_initializer),
          partitioner=self._partitioners.get(self.GAMMA_H),
          regularizer=self._regularizers.get(self.GAMMA_H))
    if self._use_batch_norm_x:
      self._gamma_x = tf.get_variable(
          self.GAMMA_X,
          shape=[4 * self._hidden_size],
          dtype=dtype,
          initializer=self._initializers.get(self.GAMMA_X, gamma_initializer),
          partitioner=self._partitioners.get(self.GAMMA_X),
          regularizer=self._regularizers.get(self.GAMMA_X))
    if self._use_batch_norm_c:
      self._gamma_c = tf.get_variable(
          self.GAMMA_C,
          shape=[self._hidden_size],
          dtype=dtype,
          initializer=self._initializers.get(self.GAMMA_C, gamma_initializer),
          partitioner=self._partitioners.get(self.GAMMA_C),
          regularizer=self._regularizers.get(self.GAMMA_C))
      self._beta_c = tf.get_variable(
          self.BETA_C,
          shape=[self._hidden_size],
          dtype=dtype,
          initializer=self._initializers.get(self.BETA_C),
          partitioner=self._partitioners.get(self.BETA_C),
          regularizer=self._regularizers.get(self.BETA_C))

Response: Initialize the variables used for the BatchNorms.


Instruction:
Can you implement a function in Python 3 that
initializes the variables used for the gates.

Response:
def _create_gate_variables(self, input_shape, dtype):
    """Initialize the variables used for the gates."""
    if len(input_shape) != 2:
      raise ValueError(
          "Rank of shape must be {} not: {}".format(2, len(input_shape)))
    input_size = input_shape.dims[1].value

    b_shape = [4 * self._hidden_size]

    equiv_input_size = self._hidden_size + input_size
    initializer = basic.create_linear_initializer(equiv_input_size)

    if self._use_batch_norm_h or self._use_batch_norm_x:
      self._w_h = tf.get_variable(
          self.W_GATES + "_H",
          shape=[self._hidden_size, 4 * self._hidden_size],
          dtype=dtype,
          initializer=self._initializers.get(self.W_GATES, initializer),
          partitioner=self._partitioners.get(self.W_GATES),
          regularizer=self._regularizers.get(self.W_GATES))
      self._w_x = tf.get_variable(
          self.W_GATES + "_X",
          shape=[input_size, 4 * self._hidden_size],
          dtype=dtype,
          initializer=self._initializers.get(self.W_GATES, initializer),
          partitioner=self._partitioners.get(self.W_GATES),
          regularizer=self._regularizers.get(self.W_GATES))
    else:
      self._w_xh = tf.get_variable(
          self.W_GATES,
          shape=[self._hidden_size + input_size, 4 * self._hidden_size],
          dtype=dtype,
          initializer=self._initializers.get(self.W_GATES, initializer),
          partitioner=self._partitioners.get(self.W_GATES),
          regularizer=self._regularizers.get(self.W_GATES))
    self._b = tf.get_variable(
        self.B_GATES,
        shape=b_shape,
        dtype=dtype,
        initializer=self._initializers.get(self.B_GATES, initializer),
        partitioner=self._partitioners.get(self.B_GATES),
        regularizer=self._regularizers.get(self.B_GATES))


Instruction:
Here you have a function in Python 3, explain what it does
def initial_state(self, batch_size, dtype=tf.float32, trainable=False,
                    trainable_initializers=None, trainable_regularizers=None,
                    name=None):
    """Builds the default start state tensor of zeros.

    Args:
      batch_size: An int, float or scalar Tensor representing the batch size.
      dtype: The data type to use for the state.
      trainable: Boolean that indicates whether to learn the initial state.
      trainable_initializers: An optional pair of initializers for the
          initial hidden state and cell state.
      trainable_regularizers: Optional regularizer function or nested structure
        of functions with the same structure as the `state_size` property of the
        core, to be used as regularizers of the initial state variable. A
        regularizer should be a function that takes a single `Tensor` as an
        input and returns a scalar `Tensor` output, e.g. the L1 and L2
        regularizers in `tf.contrib.layers`.
      name: Optional string used to prefix the initial state variable names, in
          the case of a trainable initial state. If not provided, defaults to
          the name of the module.

    Returns:
      A tensor tuple `([batch_size, state_size], [batch_size, state_size], ?)`
      filled with zeros, with the third entry present when batch norm is enabled
      with `max_unique_stats > 1', with value `0` (representing the time step).
    """
    if self._max_unique_stats == 1:
      return super(BatchNormLSTM, self).initial_state(
          batch_size, dtype=dtype, trainable=trainable,
          trainable_initializers=trainable_initializers,
          trainable_regularizers=trainable_regularizers, name=name)
    else:
      with tf.name_scope(self._initial_state_scope(name)):
        if not trainable:
          state = self.zero_state(batch_size, dtype)
        else:
          # We have to manually create the state ourselves so we don't create a
          # variable that never gets used for the third entry.
          state = rnn_core.trainable_initial_state(
              batch_size,
              (tf.TensorShape([self._hidden_size]),
               tf.TensorShape([self._hidden_size])),
              dtype=dtype,
              initializers=trainable_initializers,
              regularizers=trainable_regularizers,
              name=self._initial_state_scope(name))
        return (state[0], state[1], tf.constant(0, dtype=tf.int32))

Response: Builds the default start state tensor for the batch norm.


Instruction:
Explain what the following Python 3 code does
def state_size(self):
    if self._max_unique_stats == 1:
      return (tf.TensorShape([self._hidden_size]),
              tf.TensorShape([self._hidden_size]))
    else:
      return (tf.TensorShape([self._hidden_size]),
              tf.TensorShape([self._hidden_size]),
              tf.TensorShape(1))

Response: Tuple of tf. TensorShape s indicating the size of state tensors.


Instruction:
Write a Python 3 script to
return a new convolution.

Response:
def _new_convolution(self, use_bias):
    """Returns new convolution.

    Args:
      use_bias: Use bias in convolutions. If False, clean_dict removes bias
        entries from initializers, partitioners and regularizers passed to
        the constructor of the convolution.
    """
    def clean_dict(input_dict):
      if input_dict and not use_bias:
        cleaned_dict = input_dict.copy()
        cleaned_dict.pop("b", None)
        return cleaned_dict
      return input_dict
    return self._conv_class(
        output_channels=4*self._output_channels,
        kernel_shape=self._kernel_shape,
        stride=self._stride,
        rate=self._rate,
        padding=self._padding,
        use_bias=use_bias,
        initializers=clean_dict(self._initializers),
        partitioners=clean_dict(self._partitioners),
        regularizers=clean_dict(self._regularizers),
        name="conv")


Instruction:
Can you generate the documentation for the following Python 3 function
def _build(self, inputs, prev_state):
    input_size = inputs.get_shape()[1]
    weight_shape = (input_size, self._hidden_size)
    u_shape = (self._hidden_size, self._hidden_size)
    bias_shape = (self._hidden_size,)

    self._wz = tf.get_variable(GRU.WZ, weight_shape, dtype=inputs.dtype,
                               initializer=self._initializers.get(GRU.WZ),
                               partitioner=self._partitioners.get(GRU.WZ),
                               regularizer=self._regularizers.get(GRU.WZ))
    self._uz = tf.get_variable(GRU.UZ, u_shape, dtype=inputs.dtype,
                               initializer=self._initializers.get(GRU.UZ),
                               partitioner=self._partitioners.get(GRU.UZ),
                               regularizer=self._regularizers.get(GRU.UZ))
    self._bz = tf.get_variable(GRU.BZ, bias_shape, dtype=inputs.dtype,
                               initializer=self._initializers.get(GRU.BZ),
                               partitioner=self._partitioners.get(GRU.BZ),
                               regularizer=self._regularizers.get(GRU.BZ))
    z = tf.sigmoid(tf.matmul(inputs, self._wz) +
                   tf.matmul(prev_state, self._uz) + self._bz)

    self._wr = tf.get_variable(GRU.WR, weight_shape, dtype=inputs.dtype,
                               initializer=self._initializers.get(GRU.WR),
                               partitioner=self._partitioners.get(GRU.WR),
                               regularizer=self._regularizers.get(GRU.WR))
    self._ur = tf.get_variable(GRU.UR, u_shape, dtype=inputs.dtype,
                               initializer=self._initializers.get(GRU.UR),
                               partitioner=self._partitioners.get(GRU.UR),
                               regularizer=self._regularizers.get(GRU.UR))
    self._br = tf.get_variable(GRU.BR, bias_shape, dtype=inputs.dtype,
                               initializer=self._initializers.get(GRU.BR),
                               partitioner=self._partitioners.get(GRU.BR),
                               regularizer=self._regularizers.get(GRU.BR))
    r = tf.sigmoid(tf.matmul(inputs, self._wr) +
                   tf.matmul(prev_state, self._ur) + self._br)

    self._wh = tf.get_variable(GRU.WH, weight_shape, dtype=inputs.dtype,
                               initializer=self._initializers.get(GRU.WH),
                               partitioner=self._partitioners.get(GRU.WH),
                               regularizer=self._regularizers.get(GRU.WH))
    self._uh = tf.get_variable(GRU.UH, u_shape, dtype=inputs.dtype,
                               initializer=self._initializers.get(GRU.UH),
                               partitioner=self._partitioners.get(GRU.UH),
                               regularizer=self._regularizers.get(GRU.UH))
    self._bh = tf.get_variable(GRU.BH, bias_shape, dtype=inputs.dtype,
                               initializer=self._initializers.get(GRU.BH),
                               partitioner=self._partitioners.get(GRU.BH),
                               regularizer=self._regularizers.get(GRU.BH))
    h_twiddle = tf.tanh(tf.matmul(inputs, self._wh) +
                        tf.matmul(r * prev_state, self._uh) + self._bh)

    state = (1 - z) * prev_state + z * h_twiddle
    return state, state

Response: Connects the GRU module into the graph.


Instruction:
Can you tell what is the following Python 3 function doing
def get_possible_initializer_keys(cls, num_layers):
    keys = [cls.WT, cls.WH]
    for layer_index in xrange(num_layers):
      layer_str = str(layer_index)
      keys += [
          cls.WT + layer_str,
          cls.BT + layer_str,
          cls.WH + layer_str,
          cls.BH + layer_str]
    return set(keys)

Response: Returns the set of possible variable initializers that may be passed to the base class constructor.


Instruction:
Can you write a function in Python 3 where it
connects the highway core module into the graph.

Response:
def _build(self, inputs, prev_state):
    """Connects the highway core module into the graph.

    Args:
      inputs: Tensor of size `[batch_size, input_size]`.
      prev_state: Tensor of size `[batch_size, hidden_size]`.

    Returns:
      A tuple (output, next_state) where `output` is a Tensor of size
      `[batch_size, hidden_size]` and `next_state` is a Tensor of size
      `[batch_size, hidden_size]`.

    Raises:
      ValueError: If connecting the module into the graph any time after the
        first time, and the inferred size of the inputs does not match previous
        invocations.
    """
    input_size = inputs.get_shape()[1]
    weight_shape = (input_size, self._hidden_size)
    u_shape = (self._hidden_size, self._hidden_size)
    bias_shape = (self._hidden_size,)

    def _get_variable(name, shape):
      return tf.get_variable(
          name,
          shape,
          dtype=inputs.dtype,
          initializer=self._initializers.get(name),
          partitioner=self._partitioners.get(name),
          regularizer=self._regularizers.get(name))

    pre_highway_wt = _get_variable(self.WT, weight_shape)
    pre_highway_wh = _get_variable(self.WH, weight_shape)
    state = prev_state
    for layer_index in xrange(self._num_layers):
      layer_str = str(layer_index)
      layer_wt = _get_variable(self.WT + layer_str, u_shape)
      layer_bt = _get_variable(self.BT + layer_str, bias_shape)
      layer_wh = _get_variable(self.WH + layer_str, u_shape)
      layer_bh = _get_variable(self.BH + layer_str, bias_shape)
      linear_t = tf.matmul(state, layer_wt) + layer_bt
      linear_h = tf.matmul(state, layer_wh) + layer_bh
      if layer_index == 0:
        linear_t += tf.matmul(inputs, pre_highway_wt)
        linear_h += tf.matmul(inputs, pre_highway_wh)
      output_t = tf.sigmoid(linear_t)
      output_h = tf.tanh(linear_h)
      state = state * (1 - output_t) + output_h * output_t

    return state, state


Instruction:
Write a Python 3 function that can
get the list flattened output sizes of a list of cores.

Response:
def _get_flat_core_sizes(cores):
  """Obtains the list flattened output sizes of a list of cores.

  Args:
    cores: list of cores to get the shapes from.

  Returns:
    List of lists that, for each core, contains the list of its output
      dimensions.
  """
  core_sizes_lists = []
  for core in cores:
    flat_output_size = nest.flatten(core.output_size)
    core_sizes_lists.append(
        [tf.TensorShape(size).as_list() for size in flat_output_size])
  return core_sizes_lists


Instruction:
Implement a Python 3 function for
converting a Tensor nest to a TensorShape nest removing batch dimension.

Response:
def _get_shape_without_batch_dimension(tensor_nest):
  """Converts Tensor nest to a TensorShape nest, removing batch dimension."""
  def _strip_batch_and_convert_to_shape(tensor):
    return tensor.get_shape()[1:]
  return nest.map_structure(_strip_batch_and_convert_to_shape, tensor_nest)


Instruction:
Can you tell what is the following Python 3 function doing
def _build(self, input_, prev_state):
    self._in_to_hidden_linear = basic.Linear(
        self._hidden_size, name="in_to_hidden",
        initializers=self._initializers.get("in_to_hidden"),
        partitioners=self._partitioners.get("in_to_hidden"),
        regularizers=self._regularizers.get("in_to_hidden"))

    self._hidden_to_hidden_linear = basic.Linear(
        self._hidden_size, name="hidden_to_hidden",
        initializers=self._initializers.get("hidden_to_hidden"),
        partitioners=self._partitioners.get("hidden_to_hidden"),
        regularizers=self._regularizers.get("hidden_to_hidden"))

    in_to_hidden = self._in_to_hidden_linear(input_)
    hidden_to_hidden = self._hidden_to_hidden_linear(prev_state)
    output = self._activation(in_to_hidden + hidden_to_hidden)

    # For VanillaRNN, the next state of the RNN is the same as the output
    return output, output

Response: Connects the VanillaRNN module into the graph.


Instruction:
How would you explain what the following Python 3 function does
def _check_cores_output_sizes(self):
    for core_sizes in zip(*tuple(_get_flat_core_sizes(self._cores))):
      first_core_list = core_sizes[0][1:]
      for i, core_list in enumerate(core_sizes[1:]):
        if core_list[1:] != first_core_list:
          raise ValueError("The outputs of the provided cores are not able "
                           "to be concatenated along the first feature "
                           "dimension. Core 0 has shape %s, whereas Core %d "
                           "has shape %s - these must only differ in the first "
                           "dimension" % (core_sizes[0], i + 1, core_list))

Response: Checks the output_sizes of the cores of the DeepRNN module.


Instruction:
Here you have a function in Python 3, explain what it does
def _build(self, inputs, prev_state, **kwargs):
    current_input = inputs
    next_states = []
    outputs = []
    recurrent_idx = 0
    concatenate = lambda *args: tf.concat(args, axis=-1)
    for i, core in enumerate(self._cores):
      if self._skip_connections and i > 0:
        current_input = nest.map_structure(concatenate, inputs, current_input)

      # Determine if this core in the stack is recurrent or not and call
      # accordingly.
      if self._is_recurrent_list[i]:
        current_input, next_state = core(current_input,
                                         prev_state[recurrent_idx],
                                         **kwargs)
        next_states.append(next_state)
        recurrent_idx += 1
      else:
        current_input = core(current_input, **kwargs)

      if self._skip_connections:
        outputs.append(current_input)

    if self._skip_connections and self._concat_final_output_if_skip:
      output = nest.map_structure(concatenate, *outputs)
    else:
      output = current_input

    self._last_output_size = _get_shape_without_batch_dimension(output)
    return output, tuple(next_states)

Response: Connects the DeepRNN module into the graph.


Instruction:
Create a Python 3 function to
build the default start state for a DeepRNN.

Response:
def initial_state(self, batch_size, dtype=tf.float32, trainable=False,
                    trainable_initializers=None, trainable_regularizers=None,
                    name=None):
    """Builds the default start state for a DeepRNN.

    Args:
      batch_size: An int, float or scalar Tensor representing the batch size.
      dtype: The data type to use for the state.
      trainable: Boolean that indicates whether to learn the initial state.
      trainable_initializers: An initializer function or nested structure of
          functions with same structure as the `state_size` property of the
          core, to be used as initializers of the initial state variable.
      trainable_regularizers: Optional regularizer function or nested structure
        of functions with the same structure as the `state_size` property of the
        core, to be used as regularizers of the initial state variable. A
        regularizer should be a function that takes a single `Tensor` as an
        input and returns a scalar `Tensor` output, e.g. the L1 and L2
        regularizers in `tf.contrib.layers`.
      name: Optional string used to prefix the initial state variable names, in
          the case of a trainable initial state. If not provided, defaults to
          the name of the module.

    Returns:
      A tensor or nested tuple of tensors with same structure and shape as the
      `state_size` property of the core.

    Raises:
      ValueError: if the number of passed initializers is not the same as the
          number of recurrent cores.
    """
    initial_state = []
    if trainable_initializers is None:
      trainable_initializers = [None] * self._num_recurrent
    if trainable_regularizers is None:
      trainable_regularizers = [None] * self._num_recurrent

    num_initializers = len(trainable_initializers)

    if num_initializers != self._num_recurrent:
      raise ValueError("The number of initializers and recurrent cores should "
                       "be the same. Received %d initializers for %d specified "
                       "recurrent cores."
                       % (num_initializers, self._num_recurrent))

    with tf.name_scope(self._initial_state_scope(name)):
      recurrent_idx = 0
      for is_recurrent, core in zip(self._is_recurrent_list, self._cores):
        if is_recurrent:
          core_initial_state = core.initial_state(
              batch_size, dtype=dtype, trainable=trainable,
              trainable_initializers=trainable_initializers[recurrent_idx],
              trainable_regularizers=trainable_regularizers[recurrent_idx])
          initial_state.append(core_initial_state)
          recurrent_idx += 1
    return tuple(initial_state)


Instruction:
Write a Python 3 script to
connect the ModelRNN module into the graph.

Response:
def _build(self, inputs, prev_state):
    """Connects the ModelRNN module into the graph.

    If this is not the first time the module has been connected to the graph,
    the Tensors provided as input_ and state must have the same final
    dimension, in order for the existing variables to be the correct size for
    their corresponding multiplications. The batch size may differ for each
    connection.

    Args:
      inputs: Tensor input to the ModelRNN (ignored).
      prev_state: Tensor of size `model.output_size`.

    Returns:
      output: Tensor of size `model.output_size`.
      next_state: Tensor of size `model.output_size`.
    """
    next_state = self._model(prev_state)

    # For ModelRNN, the next state of the RNN is the same as the output
    return next_state, next_state


Instruction:
Make a summary of the following Python 3 code
def _build(self, input_sequence, state):
    input_shape = input_sequence.get_shape()
    if input_shape[0] is None:
      raise ValueError("Time dimension of input (dim 0) must be statically"
                       "known.")
    seq_length = int(input_shape[0])
    forward_state, backward_state = state

    # Lists for the forward backward output and state.
    output_sequence_f = []
    output_sequence_b = []

    # Forward pass over the sequence.
    with tf.name_scope("forward_rnn"):
      core_state = forward_state
      for i in six.moves.range(seq_length):
        core_output, core_state = self._forward_core(
            input_sequence[i, :,], core_state)

        output_sequence_f.append((core_output, core_state))

      output_sequence_f = nest.map_structure(
          lambda *vals: tf.stack(vals), *output_sequence_f)

    # Backward pass over the sequence.
    with tf.name_scope("backward_rnn"):
      core_state = backward_state
      for i in six.moves.range(seq_length - 1, -1, -1):
        core_output, core_state = self._backward_core(
            input_sequence[i, :,], core_state)

        output_sequence_b.append((core_output, core_state))

      output_sequence_b = nest.map_structure(
          lambda *vals: tf.stack(vals), *output_sequence_b)

    # Compose the full output and state sequeneces.
    return {
        "outputs": {
            "forward": output_sequence_f[0],
            "backward": output_sequence_b[0]
        },
        "state": {
            "forward": output_sequence_f[1],
            "backward": output_sequence_b[1]
        }
    }

Response: Connects the BidirectionalRNN module into the graph.


Instruction:
Given the following Python 3 function, write the documentation
def initial_state(self, batch_size, dtype=tf.float32, trainable=False,
                    trainable_initializers=None, trainable_regularizers=None,
                    name=None):
    """Builds the default start state for a BidirectionalRNN.

    The Bidirectional RNN flattens the states of its forward and backward cores
    and concatentates them.

    Args:
      batch_size: An int, float or scalar Tensor representing the batch size.
      dtype: The data type to use for the state.
      trainable: Boolean that indicates whether to learn the initial state.
      trainable_initializers: An initializer function or nested structure of
          functions with same structure as the `state_size` property of the
          core, to be used as initializers of the initial state variable.
      trainable_regularizers: Optional regularizer function or nested structure
        of functions with the same structure as the `state_size` property of the
        core, to be used as regularizers of the initial state variable. A
        regularizer should be a function that takes a single `Tensor` as an
        input and returns a scalar `Tensor` output, e.g. the L1 and L2
        regularizers in `tf.contrib.layers`.
      name: Optional string used to prefix the initial state variable names, in
          the case of a trainable initial state. If not provided, defaults to
          the name of the module.

    Returns:
      Tuple of initial states from forward and backward RNNs.
    """
    name = "state" if name is None else name
    forward_initial_state = self._forward_core.initial_state(
        batch_size, dtype, trainable, trainable_initializers,
        trainable_regularizers, name=name+"_forward")
    backward_initial_state = self._backward_core.initial_state(
        batch_size, dtype, trainable, trainable_initializers,
        trainable_regularizers, name=name+"_backward")
    return forward_initial_state, backward_initial_state

Response: Builds the default start state for a BidirectionalRNN.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _build(self, inputs, is_training=True, dropout_keep_prob=0.5):
    self._input_shape = tuple(inputs.get_shape().as_list())
    net = inputs

    final_index = self._num_layers - 1
    for layer_id in xrange(self._num_layers):
      net = self._layers[layer_id](net)

      if final_index != layer_id or self._activate_final:
        # Only perform dropout whenever we are activating the layer's outputs.
        if self._use_dropout:
          keep_prob = utils.smart_cond(
              is_training, true_fn=lambda: dropout_keep_prob,
              false_fn=lambda: tf.constant(1.0)
          )
          net = tf.nn.dropout(net, keep_prob=keep_prob)
        net = self._activation(net)

    return net

Response: Assembles the MLP and connects it to the graph.


Instruction:
How would you explain what the following Python 3 function does
def output_sizes(self):
    return tuple([l() if callable(l) else l for l in self._output_sizes])

Response: Returns a tuple of all the output sizes of all the layers.


Instruction:
Write a Python 3 function for
returning a new MLP with the same parameters as the current one.

Response:
def transpose(self, name=None, activate_final=None):
    """Returns transposed `MLP`.

    Args:
      name: Optional string specifying the name of the transposed module. The
        default name is constructed by appending "_transpose"
        to `self.module_name`.
      activate_final: Optional boolean determining if the activation and batch
        normalization, if turned on, are applied to the final layer.

    Returns:
      Matching transposed `MLP` module.
    """
    if name is None:
      name = self.module_name + "_transpose"
    if activate_final is None:
      activate_final = self.activate_final
    output_sizes = [lambda l=layer: l.input_shape[1] for layer in self._layers]
    output_sizes.reverse()
    return MLP(
        name=name,
        output_sizes=output_sizes,
        activation=self.activation,
        activate_final=activate_final,
        initializers=self.initializers,
        partitioners=self.partitioners,
        regularizers=self.regularizers,
        use_bias=self.use_bias,
        use_dropout=self.use_dropout)


Instruction:
Implement a Python 3 function for
creating a new MLP with the same structure.

Response:
def clone(self, name=None):
    """Creates a new MLP with the same structure.

    Args:
      name: Optional string specifying the name of the new module. The default
        name is constructed by appending "_clone" to the original name.

    Returns:
      A cloned `MLP` module.
    """

    if name is None:
      name = self.module_name + "_clone"
    return MLP(
        name=name,
        output_sizes=self.output_sizes,
        activation=self.activation,
        activate_final=self.activate_final,
        initializers=self.initializers,
        partitioners=self.partitioners,
        regularizers=self.regularizers,
        use_bias=self.use_bias,
        use_dropout=self.use_dropout)


Instruction:
Write a Python 3 script for
calculating the minimum size of the input layer.

Response:
def _calc_min_size(self, conv_layers):
    """Calculates the minimum size of the input layer.

    Given a set of convolutional layers, calculate the minimum value of
    the `input_height` and `input_width`, i.e. such that the output has
    size 1x1. Assumes snt.VALID padding.

    Args:
      conv_layers: List of tuples `(output_channels, (kernel_size, stride),
        (pooling_size, pooling_stride))`

    Returns:
      Minimum value of input height and width.
    """
    input_size = 1

    for _, conv_params, max_pooling in reversed(conv_layers):
      if max_pooling is not None:
        kernel_size, stride = max_pooling
        input_size = input_size * stride + (kernel_size - stride)

      if conv_params is not None:
        kernel_size, stride = conv_params
        input_size = input_size * stride + (kernel_size - stride)

    return input_size


Instruction:
Implement a function in Python 3 to
connect the AlexNet module into the graph.

Response:
def _build(self, inputs, keep_prob=None, is_training=None,
             test_local_stats=True):
    """Connects the AlexNet module into the graph.

    The is_training flag only controls the batch norm settings, if `False` it
    does not force no dropout by overriding any input `keep_prob`. To avoid any
    confusion this may cause, if `is_training=False` and `keep_prob` would cause
    dropout to be applied, an error is thrown.

    Args:
      inputs: A Tensor of size [batch_size, input_height, input_width,
        input_channels], representing a batch of input images.
      keep_prob: A scalar Tensor representing the dropout keep probability.
        When `is_training=False` this must be None or 1 to give no dropout.
      is_training: Boolean to indicate if we are currently training. Must be
          specified if batch normalization or dropout is used.
      test_local_stats: Boolean to indicate to `snt.BatchNorm` if batch
        normalization should  use local batch statistics at test time.
        By default `True`.

    Returns:
      A Tensor of size [batch_size, output_size], where `output_size` depends
      on the mode the network was constructed in.

    Raises:
      base.IncompatibleShapeError: If any of the input image dimensions
        (input_height, input_width) are too small for the given network mode.
      ValueError: If `keep_prob` is not None or 1 when `is_training=False`.
      ValueError: If `is_training` is not explicitly specified when using
        batch normalization.
    """
    # Check input shape
    if (self._use_batch_norm or keep_prob is not None) and is_training is None:
      raise ValueError("Boolean is_training flag must be explicitly specified "
                       "when using batch normalization or dropout.")

    input_shape = inputs.get_shape().as_list()
    if input_shape[1] < self._min_size or input_shape[2] < self._min_size:
      raise base.IncompatibleShapeError(
          "Image shape too small: ({:d}, {:d}) < {:d}".format(
              input_shape[1], input_shape[2], self._min_size))

    net = inputs

    # Check keep prob
    if keep_prob is not None:
      valid_inputs = tf.logical_or(is_training, tf.equal(keep_prob, 1.))
      keep_prob_check = tf.assert_equal(
          valid_inputs, True,
          message="Input `keep_prob` must be None or 1 if `is_training=False`.")
      with tf.control_dependencies([keep_prob_check]):
        net = tf.identity(net)

    for i, params in enumerate(self._conv_layers):
      output_channels, conv_params, max_pooling = params

      kernel_size, stride = conv_params

      conv_mod = conv.Conv2D(
          name="conv_{}".format(i),
          output_channels=output_channels,
          kernel_shape=kernel_size,
          stride=stride,
          padding=conv.VALID,
          initializers=self._initializers,
          partitioners=self._partitioners,
          regularizers=self._regularizers)

      if not self.is_connected:
        self._conv_modules.append(conv_mod)

      net = conv_mod(net)

      if self._use_batch_norm:
        bn = batch_norm.BatchNorm(**self._batch_norm_config)
        net = bn(net, is_training, test_local_stats)

      net = tf.nn.relu(net)

      if max_pooling is not None:
        pooling_kernel_size, pooling_stride = max_pooling
        net = tf.nn.max_pool(
            net,
            ksize=[1, pooling_kernel_size, pooling_kernel_size, 1],
            strides=[1, pooling_stride, pooling_stride, 1],
            padding=conv.VALID)

    net = basic.BatchFlatten(name="flatten")(net)

    for i, output_size in enumerate(self._fc_layers):
      linear_mod = basic.Linear(
          name="fc_{}".format(i),
          output_size=output_size,
          initializers=self._initializers,
          partitioners=self._partitioners)

      if not self.is_connected:
        self._linear_modules.append(linear_mod)

      net = linear_mod(net)

      if self._use_batch_norm and self._bn_on_fc_layers:
        bn = batch_norm.BatchNorm(**self._batch_norm_config)
        net = bn(net, is_training, test_local_stats)

      net = tf.nn.relu(net)

      if keep_prob is not None:
        net = tf.nn.dropout(net, keep_prob=keep_prob)

    return net


Instruction:
Can you implement a function in Python 3 that
generates one input sequence and output label.

Response:
def _get_single_set(self, num_objects, num_features):
    """Generate one input sequence and output label.

    Each sequences of objects has a feature that consists of the feature vector
    for that object plus the encoding for its ID, the reference vector ID and
    the n-th value relative ID for a total feature size of:

      `num_objects` * 3  + `num_features`

    Args:
      num_objects: int. number of objects in the sequence.
      num_features: int. feature size of each object.

    Returns:
      1. np.ndarray (`num_objects`, (`num_features` + 3 * `num_objects`)).
      2. np.ndarray (1,). Output object reference label.
    """
    # Generate random binary vectors
    data = np.random.uniform(-1, 1, size=(num_objects, num_features))

    distances = spdistance.squareform(spdistance.pdist(data))
    distance_idx = np.argsort(distances)

    # Choose random distance
    nth = np.random.randint(0, num_objects)

    # Pick out the nth furthest for each object
    nth_furthest = distance_idx[:, nth]

    # Choose random reference object
    reference = np.random.randint(0, num_objects)

    # Get identity of object that is the nth furthest from reference object
    labels = nth_furthest[reference]

    # Compile data
    object_ids = np.identity(num_objects)
    nth_matrix = np.zeros((num_objects, num_objects))
    nth_matrix[:, nth] = 1
    reference_object = np.zeros((num_objects, num_objects))
    reference_object[:, reference] = 1

    inputs = np.concatenate([data, object_ids, reference_object, nth_matrix],
                            axis=-1)
    inputs = np.random.permutation(inputs)
    labels = np.expand_dims(labels, axis=0)
    return inputs.astype(np.float32), labels.astype(np.float32)


Instruction:
How would you implement a function in Python 3 that
assembles a batch of input tensors and output labels.

Response:
def _get_batch_data(self, batch_size, num_objects, num_features):
    """Assembles a batch of input tensors and output labels.

    Args:
      batch_size: int. number of sequence batches.
      num_objects: int. number of objects in the sequence.
      num_features: int. feature size of each object.

    Returns:
      1. np.ndarray (`batch_size`, `num_objects`,
                     (`num_features` + 3 * `num_objects`)).
      2. np.ndarray (`batch_size`). Output object reference label.
    """
    all_inputs = []
    all_labels = []
    for _ in six.moves.range(batch_size):
      inputs, labels = self._get_single_set(num_objects, num_features)
      all_inputs += [inputs]
      all_labels += [labels]
    input_data = np.concatenate(all_inputs, axis=0)
    label_data = np.concatenate(all_labels, axis=0)
    return input_data, label_data


Instruction:
Given the following Python 3 function, write the documentation
def get_batch(self):
    params = [self._batch_size, self._num_objects, self._num_features]
    inputs, labels = tf.py_func(self._get_batch_data, params,
                                [tf.float32, tf.float32])
    inputs = tf.reshape(inputs, [self._batch_size, self._num_objects,
                                 self._num_features + self._num_objects * 3])
    labels = tf.reshape(labels, [-1])
    return inputs, labels

Response: Returns set of nth - farthest input tensors and labels.


Instruction:
Given the following Python 3 function, write the documentation
def _default_transpose_size(input_shape, stride, kernel_shape=None,
                            padding=SAME):
  """Returns default (maximal) output shape for a transpose convolution.

  In general, there are multiple possible output shapes that a transpose
  convolution with a given `input_shape` can map to. This function returns the
  output shape which evenly divides the stride to produce the input shape in
  a forward convolution, i.e. the maximal valid output shape with the given
  configuration:

  if the padding type is SAME then:  output_shape = input_shape * stride
  if the padding type is VALID then: output_shape = input_shape * stride +
                                                    kernel_shape - 1

  See the following documentation for an explanation of VALID versus SAME
  padding modes:
  https://www.tensorflow.org/versions/r0.8/api_docs/python/nn.html#convolution

  Args:
    input_shape: Sequence of sizes of each dimension of the input, excluding
      batch and channel dimensions.
    stride: Sequence or integer of kernel strides, excluding batch and channel
      dimension strides.
    kernel_shape: Sequence or integer of kernel sizes.
    padding: Padding algorithm, either `snt.SAME` or `snt.VALID`.

  Returns:
    output_shape: A tuple of sizes for a transposed convolution that divide
      evenly with the given strides, kernel shapes, and padding algorithm.
  """
  if not input_shape:
    raise TypeError("input_shape is None; if using Sonnet, are you sure you "
                    "have connected the module to inputs?")
  input_length = len(input_shape)
  stride = _fill_and_verify_parameter_shape(stride, input_length, "stride")
  padding = _verify_conv_op_supported_padding(padding)

  output_shape = tuple(x * y for x, y in zip(input_shape, stride))

  if padding == VALID:
    kernel_shape = _fill_and_verify_parameter_shape(kernel_shape, input_length,
                                                    "kernel")
    output_shape = tuple(x + y - 1 for x, y in zip(output_shape, kernel_shape))

  return output_shape

Response: Returns the default output shape for a transpose convolution.


Instruction:
Implement a function in Python 3 to
convert a dimension to a tuple of dimensions of a given size.

Response:
def _fill_shape(x, n):
  """Converts a dimension to a tuple of dimensions of a given size.

  This is used to allow shorthand notation for various configuration parameters.
  A user can provide either, for example, `2` or `[2, 2]` as a kernel shape, and
  this function returns `(2, 2)` in both cases. Passing `[1, 2]` will return
  `(1, 2)`.

  Args:
    x: An integer, tf.Dimension, or an iterable of them.
    n: An integer, the size of the desired output list

  Returns:
    If `x` is an integer, a tuple of size `n` containing `n` copies of `x`.
    If `x` is an iterable of integers or tf.Dimension of size `n`, it returns
      `tuple(x)`.

  Raises:
    TypeError: If n is not a positive integer;
      or if x is neither integer nor an iterable of size n.
  """
  if not isinstance(n, numbers.Integral) or n < 1:
    raise TypeError("n must be a positive integer")

  if (isinstance(x, numbers.Integral) or isinstance(x, tf.Dimension)) and x > 0:
    return (x,) * n

  try:
    if len(x) == n and all(v > 0 for v in x):
      return tuple(x)
  except TypeError:
    pass

  raise TypeError("x is {}, must be either a positive integer "
                  "or an iterable of positive integers of size {}"
                  .format(x, n))


Instruction:
Can you tell what is the following Python 3 function doing
def _fill_and_verify_parameter_shape(x, n, parameter_label):
  try:
    return _fill_shape(x, n)
  except TypeError as e:
    raise base.IncompatibleShapeError("Invalid " + parameter_label + " shape: "
                                      "{}".format(e))

Response: Expands x if necessary into a n - D kernel shape and reports errors.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _fill_and_verify_padding(padding, n):
  if not isinstance(n, numbers.Integral) or n < 1:
    raise TypeError("n must be a positive integer")

  if isinstance(padding, six.string_types) and padding in ALLOWED_PADDINGS:
    return (padding,) * n

  try:
    if len(padding) == n and all(p in ALLOWED_PADDINGS for p in padding):
      return tuple(padding)
  except TypeError:
    pass

  raise TypeError("padding is {}, must be member of '{}' or an iterable of "
                  "these of size {}".format(padding, ALLOWED_PADDINGS, n))

Response: Verifies that the provided padding is supported and expands to size n.


Instruction:
How would you implement a function in Python 3 that
returns a boolean indicating whether to use SAME or VALID for the underlying convolution op.

Response:
def _padding_to_conv_op_padding(padding):
  """Whether to use SAME or VALID for the underlying convolution op.

  Args:
    padding: A tuple of members of ALLOWED_PADDINGS, e.g. as returned from
      `_fill_and_verify_padding`.

  Returns:
    One of CONV_OP_ALLOWED_PADDINGS, the padding method to use for the
    underlying convolution op.

  Raises:
    ValueError: If padding is not a tuple.
  """
  if not isinstance(padding, tuple):
    raise ValueError("padding should be a tuple.")
  if all(p == SAME for p in padding):
    # If we want SAME padding for all dimensions then we can use SAME for the
    # conv and avoid doing any extra padding.
    return SAME
  else:
    # Otherwise we prefer to use VALID, since we can implement all the other
    # padding types just by adding some extra padding before doing a VALID conv.
    # (We could use SAME but then we'd also have to crop outputs in some cases).
    return VALID


Instruction:
Implement a Python 3 function for
expanding the provided stride to size n and pads it with 1s.

Response:
def _fill_and_one_pad_stride(stride, n, data_format=DATA_FORMAT_NHWC):
  """Expands the provided stride to size n and pads it with 1s."""
  if isinstance(stride, numbers.Integral) or (
      isinstance(stride, collections.Iterable) and len(stride) <= n):
    if data_format.startswith("NC"):
      return (1, 1,) + _fill_shape(stride, n)
    elif data_format.startswith("N") and data_format.endswith("C"):
      return (1,) + _fill_shape(stride, n) + (1,)
    else:
      raise ValueError(
          "Invalid data_format {:s}. Must start with N and have a channel dim "
          "either follow the N dim or come at the end".format(data_format))
  elif isinstance(stride, collections.Iterable) and len(stride) == n + 2:
    return stride
  else:
    raise base.IncompatibleShapeError(
        "stride is {} ({}), must be either a positive integer or an iterable of"
        " positive integers of size {}".format(stride, type(stride), n))


Instruction:
Write a Python 3 function that can
verify that the inputs are semantically correct.

Response:
def _verify_inputs(inputs, channel_index, data_format):
  """Verifies `inputs` is semantically correct.

  Args:
    inputs: An input tensor provided by the user.
    channel_index: The index of the channel dimension.
    data_format: The format of the data in `inputs`.

  Raises:
    base.IncompatibleShapeError: If the shape of `inputs` doesn't match
      `data_format`.
    base.UnderspecifiedError: If the channel dimension of `inputs` isn't
      defined.
    TypeError: If input Tensor dtype is not compatible with either
      `tf.float16`, `tf.bfloat16` or `tf.float32`.
  """
  # Check shape.
  input_shape = tuple(inputs.get_shape().as_list())
  if len(input_shape) != len(data_format):
    raise base.IncompatibleShapeError((
        "Input Tensor must have rank {} corresponding to "
        "data_format {}, but instead was {} of rank {}.").format(
            len(data_format), data_format, input_shape, len(input_shape)))

  # Check type.
  if not (tf.float16.is_compatible_with(inputs.dtype) or
          tf.bfloat16.is_compatible_with(inputs.dtype) or
          tf.float32.is_compatible_with(inputs.dtype)):
    raise TypeError(
        "Input must have dtype tf.float16, tf.bfloat16 or tf.float32, "
        "but dtype was {}".format(inputs.dtype))

  # Check channel dim.
  input_channels = input_shape[channel_index]
  if input_channels is None:
    raise base.UnderspecifiedError(
        "Number of input channels must be known at module build time")


Instruction:
Write a Python 3 script for
returning a default initializer for the weights of a convolutional module.

Response:
def create_weight_initializer(fan_in_shape, dtype=tf.float32):
  """Returns a default initializer for the weights of a convolutional module."""
  stddev = 1 / math.sqrt(np.prod(fan_in_shape))
  return tf.truncated_normal_initializer(stddev=stddev, dtype=dtype)


Instruction:
Make a summary of the following Python 3 code
def _find_channel_index(data_format):
  for i, c in enumerate(data_format):
    if c == "C":
      return i
  raise ValueError("data_format requires a channel dimension. Got: {}"
                   .format(data_format))

Response: Returns the index of the channel dimension.


Instruction:
Can you generate the documentation for the following Python 3 function
def _apply_bias(inputs, outputs, channel_index, data_format, output_channels,
                initializers, partitioners, regularizers):
  """Initialize and apply a bias to the outputs.

  Figures out the shape of the bias vector, initialize it, and applies it.

  Args:
    inputs: A Tensor of shape `data_format`.
    outputs: A Tensor of shape `data_format`.
    channel_index: The index of the channel dimension in `inputs`.
    data_format: Format of `inputs`.
    output_channels: Channel dimensionality for `outputs`.
    initializers: Optional dict containing ops to initialize the biases
      (with key 'b').
    partitioners: Optional dict containing partitioners to partition the
      biases (with key 'b').
    regularizers: Optional dict containing regularizers for the biases
      (with key 'b').

  Returns:
    b: The constructed bias variable.
    outputs: The `outputs` argument that has had a bias applied.
  """
  bias_shape = (output_channels,)
  if "b" not in initializers:
    initializers["b"] = create_bias_initializer(bias_shape,
                                                dtype=inputs.dtype)
  b = tf.get_variable("b",
                      shape=bias_shape,
                      dtype=inputs.dtype,
                      initializer=initializers["b"],
                      partitioner=partitioners.get("b", None),
                      regularizer=regularizers.get("b", None))

  # tf.nn.bias_add only supports 2 data formats.
  if data_format in (DATA_FORMAT_NHWC, DATA_FORMAT_NCHW):
    # Supported as-is.
    outputs = tf.nn.bias_add(outputs, b, data_format=data_format)
  else:
    # Create our own bias vector.
    bias_correct_dim = [1] * len(data_format)
    bias_correct_dim[channel_index] = output_channels
    outputs += tf.reshape(b, bias_correct_dim)

  return b, outputs

Response: Initialize and apply a bias to the outputs.


Instruction:
Here you have a function in Python 3, explain what it does
def _build(self, inputs):
    _verify_inputs(inputs, self._channel_index, self._data_format)
    self._input_shape = tuple(inputs.get_shape().as_list())
    self._input_channels = self._input_shape[self._channel_index]

    self._w = self._construct_w(inputs)

    if self._mask is not None:
      w = self._apply_mask()
    else:
      w = self._w

    inputs = self._pad_input(inputs)
    outputs = self._apply_conv(inputs, w)

    if self._use_bias:
      self._b, outputs = _apply_bias(
          inputs, outputs, self._channel_index, self._data_format,
          self.output_channels, self._initializers, self._partitioners,
          self._regularizers)

    return outputs

Response: Connects the _ConvND module into the graph with input Tensor inputs.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _pad_input(self, inputs):
    if all(p == self._conv_op_padding for p in self._padding):
      # All axes require the same padding type that we're going to use for the
      # underlying convolution op, so nothing needs to be done:
      return inputs

    # In all other cases we use VALID as the underlying padding type, and for
    # the axes which require something other than VALID, we pad inputs ourselves
    # before the convolution.
    assert self._conv_op_padding == VALID

    def pad_amount(kernel_size, rate, padding):
      """Pre- and post-padding required for a particular axis before conv op."""
      # The effective kernel size includes any holes/gaps introduced by the
      # dilation rate. It's equal to kernel_size when rate == 1.
      effective_kernel_size = int((kernel_size - 1) * rate + 1)
      if padding == FULL:
        return [effective_kernel_size - 1, effective_kernel_size - 1]
      if padding == CAUSAL:
        return [effective_kernel_size - 1, 0]
      if padding == REVERSE_CAUSAL:
        return [0, effective_kernel_size - 1]
      if padding == SAME:
        return [(effective_kernel_size - 1) // 2, effective_kernel_size // 2]
      # padding == VALID
      return [0, 0]

    paddings = map(pad_amount, self._kernel_shape, self._rate, self._padding)
    if self._data_format.startswith("NC"):  # N, C, ...
      paddings = [[0, 0], [0, 0]] + list(paddings)
    else:  # N, ..., C
      paddings = [[0, 0]] + list(paddings) + [[0, 0]]

    return tf.pad(inputs, paddings)

Response: Pads the input tensor with the desired padding type.


Instruction:
Write a Python 3 script for
applying a convolution operation on inputs using variable w.

Response:
def _apply_conv(self, inputs, w):
    """Apply a convolution operation on `inputs` using variable `w`.

    Args:
      inputs: A Tensor of shape `data_format` and of type `tf.float16`,
          `tf.bfloat16` or `tf.float32`.
      w: A weight matrix of the same type as `inputs`.

    Returns:
      outputs: The result of the convolution operation on `inputs`.
    """
    outputs = tf.nn.convolution(inputs, w, strides=self._stride,
                                padding=self._conv_op_padding,
                                dilation_rate=self._rate,
                                data_format=self._data_format)
    return outputs


Instruction:
Can you write a function in Python 3 where it
applies the passed - in mask to the convolution matrix.

Response:
def _apply_mask(self):
    """Applies the passed-in mask to the convolution matrix.

    Returns:
      w: A copy of the convolution matrix that has had the mask applied.

    Raises:
      base.IncompatibleShapeError: If the mask shape has more dimensions than
          the weight matrix.
      base.IncompatibleShapeError: If the mask and the weight matrix don't
          match on shape.
    """
    w = self._w
    w_shape = w.get_shape()
    mask_shape = self._mask.get_shape()

    if mask_shape.ndims > w_shape.ndims:
      raise base.IncompatibleShapeError(
          "Invalid mask shape: {}. Max shape: {}".format(
              mask_shape.ndims, len(self._data_format)
          )
      )
    if mask_shape != w_shape[:mask_shape.ndims]:
      raise base.IncompatibleShapeError(
          "Invalid mask shape: {}. Weight shape: {}".format(
              mask_shape, w_shape
          )
      )
    # TF broadcasting is a bit fragile.
    # Expand the shape of self._mask by one dim at a time to the right
    # until the rank matches `weight_shape`.
    while self._mask.get_shape().ndims < w_shape.ndims:
      self._mask = tf.expand_dims(self._mask, -1)

    # tf.Variable & tf.ResourceVariable don't support *=.
    w = w * self._mask  # pylint: disable=g-no-augmented-assignment

    return w


Instruction:
Write a Python 3 script for
returning the number of output channels.

Response:
def output_channels(self):
    """Returns the number of output channels."""
    if callable(self._output_channels):
      self._output_channels = self._output_channels()
    # Channel must be integer.
    self._output_channels = int(self._output_channels)
    return self._output_channels


Instruction:
Implement a function in Python 3 to
return the padding algorithm used for each SearchResult dimension.

Response:
def padding(self):
    """Returns the padding algorithm used, if this is the same for all dims.

    Use `.paddings` if you want a tuple with the padding algorithm used for each
    dimension.

    Returns:
      The padding algorithm used, if this is the same for all dimensions.

    Raises:
      ValueError: If different padding algorithms are used for different
        dimensions.
    """
    # This is for backwards compatibility -- previously only a single
    # padding setting was supported across all dimensions.
    if all(p == self._padding[0] for p in self._padding):
      return self._padding[0]
    else:
      raise ValueError("This layer uses different paddings for different "
                       "dimensions. Use .paddings if you want a tuple of "
                       "per-dimension padding settings.")


Instruction:
Can you tell what is the following Python 3 function doing
def clone(self, name=None):
    if name is None:
      name = self.module_name + "_clone"

    return type(self)(output_channels=self.output_channels,
                      kernel_shape=self._kernel_shape,
                      stride=self._stride,
                      rate=self._rate,
                      padding=self._padding,
                      use_bias=self._use_bias,
                      initializers=self._initializers,
                      partitioners=self._partitioners,
                      regularizers=self._regularizers,
                      mask=self._mask,
                      data_format=self._data_format,
                      custom_getter=self._custom_getter,
                      name=name)

Response: Returns a copy of the current module.


Instruction:
Explain what the following Python 3 code does
def _build(self, inputs):
    _verify_inputs(inputs, self._channel_index, self._data_format)
    self._input_shape = tuple(inputs.get_shape().as_list())
    self._input_channels = self._input_shape[self._channel_index]

    # First, figure out what the non-(N,C) dims will be.
    if self._use_default_output_shape:
      def _default_transpose_size_wrapper():
        if self._data_format.startswith("NC"):
          input_size = self._input_shape[2:]
          stride = self.stride[2:]
        else:  # self._data_format == N*WC
          input_size = self._input_shape[1:-1]
          stride = self.stride[1:-1]
        return _default_transpose_size(input_size,
                                       stride,
                                       kernel_shape=self._kernel_shape,
                                       padding=self._padding)

      self._output_shape = _default_transpose_size_wrapper

    if len(self.output_shape) != self._n:
      raise base.IncompatibleShapeError(
          "Output shape must have rank {}, but instead was {}".format(
              self._n, len(self.output_shape)))

    # Now, construct the size of the output, including the N + C dims.
    output_shape = self._infer_all_output_dims(inputs)

    self._w = self._construct_w(inputs)

    if self._n == 1:
      # Add a dimension for the height.
      if self._data_format == DATA_FORMAT_NWC:
        h_dim = 1
        two_dim_conv_data_format = DATA_FORMAT_NHWC
      else:  # self._data_format == DATA_FORMAT_NCW
        h_dim = 2
        two_dim_conv_data_format = DATA_FORMAT_NCHW
      inputs = tf.expand_dims(inputs, h_dim)
      two_dim_conv_stride = self.stride[:h_dim] + (1,) + self.stride[h_dim:]
      outputs = tf.nn.conv2d_transpose(inputs,
                                       self._w,
                                       output_shape,
                                       strides=two_dim_conv_stride,
                                       padding=self._padding,
                                       data_format=two_dim_conv_data_format)
      # Remove the height dimension to return a 3D tensor.
      outputs = tf.squeeze(outputs, [h_dim])
    elif self._n == 2:
      outputs = tf.nn.conv2d_transpose(inputs,
                                       self._w,
                                       output_shape,
                                       strides=self._stride,
                                       padding=self._padding,
                                       data_format=self._data_format)
    else:
      outputs = tf.nn.conv3d_transpose(inputs,
                                       self._w,
                                       output_shape,
                                       strides=self._stride,
                                       padding=self._padding,
                                       data_format=self._data_format)

    if self._use_bias:
      self._b, outputs = _apply_bias(
          inputs, outputs, self._channel_index, self._data_format,
          self._output_channels, self._initializers, self._partitioners,
          self._regularizers)

    outputs = self._recover_shape_information(inputs, outputs)
    return outputs

Response: Connects the _ConvNDTranspose module into the graph.


Instruction:
How would you explain what the following Python 3 function does
def _infer_all_output_dims(self, inputs):
    # Use tensorflow shape op to manipulate inputs shape, so that unknown batch
    # size - which can happen when using input placeholders - is handled
    # correcly.
    batch_size = tf.expand_dims(tf.shape(inputs)[0], 0)
    out_channels = (self.output_channels,)

    # Height dim needs to be added to everything for 1D Conv
    # as we'll be using the 2D Conv Transpose op.
    if self._n == 1:
      out_shape = (1,) + self.output_shape
    else:
      out_shape = self.output_shape

    if self._data_format.startswith("NC"):
      out_shape_tuple = out_channels + out_shape
    elif self._data_format.startswith("N") and self._data_format.endswith("C"):
      out_shape_tuple = out_shape + out_channels

    output_shape = tf.concat([batch_size, out_shape_tuple], 0)
    return output_shape

Response: Infer the output shape for inputs after a deconvolution.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _recover_shape_information(self, inputs, outputs):
    batch_size_value = inputs.get_shape()[0]
    if self._data_format.startswith("NC"):
      output_shape_value = ((batch_size_value, self.output_channels) +
                            self.output_shape)
    elif self._data_format.startswith("N") and self._data_format.endswith("C"):
      output_shape_value = ((batch_size_value,) + self.output_shape +
                            (self.output_channels,))
    outputs.set_shape(output_shape_value)
    return outputs

Response: Recover output tensor shape value to enable shape inference.


Instruction:
Can you generate the documentation for the following Python 3 function
def output_shape(self):
    if self._output_shape is None:
      self._ensure_is_connected()
    if callable(self._output_shape):
      self._output_shape = tuple(self._output_shape())
    return self._output_shape

Response: Returns the output shape of the current node.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def transpose(self, name=None):
    if name is None:
      name = self.module_name + "_transpose"

    if self._data_format == DATA_FORMAT_NWC:
      stride = self._stride[1:-1]
    else:  # self._data_format == DATA_FORMAT_NCW
      stride = self._stride[2:]

    return Conv1D(output_channels=lambda: self.input_channels,
                  kernel_shape=self.kernel_shape,
                  stride=stride,
                  padding=self.padding,
                  use_bias=self._use_bias,
                  initializers=self.initializers,
                  partitioners=self.partitioners,
                  regularizers=self.regularizers,
                  data_format=self._data_format,
                  custom_getter=self._custom_getter,
                  name=name)

Response: Returns matching Conv1D module.


Instruction:
How would you implement a function in Python 3 that
returns a new module with matching input channels and output channels.

Response:
def transpose(self, name=None):
    """Returns matching `Conv2DTranspose` module.

    Args:
      name: Optional string assigning name of transpose module. The default name
        is constructed by appending "_transpose" to `self.name`.

    Returns:
      `Conv2DTranspose` module.

    Raises:
     base.NotSupportedError: If `rate` in any dimension > 1.
    """
    if any(x > 1 for x in self._rate):
      raise base.NotSupportedError(
          "Cannot transpose a dilated convolution module.")

    if any(p != self._conv_op_padding for p in self._padding):
      raise base.NotSupportedError(
          "Cannot tranpose a convolution using mixed paddings or paddings "
          "other than SAME or VALID.")

    if name is None:
      name = self.module_name + "_transpose"

    def output_shape():
      if self._data_format == DATA_FORMAT_NCHW:
        return self.input_shape[2:4]
      else:  # data_format == DATA_FORMAT_NHWC
        return self.input_shape[1:3]

    return Conv2DTranspose(output_channels=lambda: self._input_channels,
                           output_shape=output_shape,
                           kernel_shape=self._kernel_shape,
                           stride=self._stride,
                           padding=self._conv_op_padding,
                           use_bias=self._use_bias,
                           initializers=self._initializers,
                           partitioners=self._partitioners,
                           regularizers=self._regularizers,
                           data_format=self._data_format,
                           custom_getter=self._custom_getter,
                           name=name)