Instruction:
Write a Python 3 function that can
grow alive sequences by one token and collect top 2 * beam_size sequences.

Response:
def _grow_alive_seq(self, state):
    """Grow alive sequences by one token, and collect top 2*beam_size sequences.

    2*beam_size sequences are collected because some sequences may have reached
    the EOS token. 2*beam_size ensures that at least beam_size sequences are
    still alive.

    Args:
      state: A dictionary with the current loop state.
    Returns:
      Tuple of
      (Top 2*beam_size sequences [batch_size, 2 * beam_size, cur_index + 1],
       Scores of returned sequences [batch_size, 2 * beam_size],
       New alive cache, for each of the 2 * beam_size sequences)
    """
    i = state[_StateKeys.CUR_INDEX]
    alive_seq = state[_StateKeys.ALIVE_SEQ]
    alive_log_probs = state[_StateKeys.ALIVE_LOG_PROBS]
    alive_cache = state[_StateKeys.ALIVE_CACHE]

    beams_to_keep = 2 * self.beam_size

    # Get logits for the next candidate IDs for the alive sequences. Get the new
    # cache values at the same time.
    flat_ids = _flatten_beam_dim(alive_seq)  # [batch_size * beam_size]
    flat_cache = nest.map_structure(_flatten_beam_dim, alive_cache)

    flat_logits, flat_cache = self.symbols_to_logits_fn(flat_ids, i, flat_cache)

    # Unflatten logits to shape [batch_size, beam_size, vocab_size]
    logits = _unflatten_beam_dim(flat_logits, self.batch_size, self.beam_size)
    new_cache = nest.map_structure(
        lambda t: _unflatten_beam_dim(t, self.batch_size, self.beam_size),
        flat_cache)

    # Convert logits to normalized log probs
    candidate_log_probs = _log_prob_from_logits(logits)

    # Calculate new log probabilities if each of the alive sequences were
    # extended # by the the candidate IDs.
    # Shape [batch_size, beam_size, vocab_size]
    log_probs = candidate_log_probs + tf.expand_dims(alive_log_probs, axis=2)

    # Each batch item has beam_size * vocab_size candidate sequences. For each
    # batch item, get the k candidates with the highest log probabilities.
    flat_log_probs = tf.reshape(log_probs,
                                [-1, self.beam_size * self.vocab_size])
    topk_log_probs, topk_indices = tf.nn.top_k(flat_log_probs, k=beams_to_keep)

    # Extract the alive sequences that generate the highest log probabilities
    # after being extended.
    topk_beam_indices = topk_indices // self.vocab_size
    topk_seq, new_cache = _gather_beams(
        [alive_seq, new_cache], topk_beam_indices, self.batch_size,
        beams_to_keep)

    # Append the most probable IDs to the topk sequences
    topk_ids = topk_indices % self.vocab_size
    topk_ids = tf.expand_dims(topk_ids, axis=2)
    topk_seq = tf.concat([topk_seq, topk_ids], axis=2)
    return topk_seq, topk_log_probs, new_cache


Instruction:
Explain what the following Python 3 code does
def _get_new_alive_state(self, new_seq, new_log_probs, new_cache):
    # To prevent finished sequences from being considered, set log probs to -INF
    new_finished_flags = tf.equal(new_seq[:, :, -1], self.eos_id)
    new_log_probs += tf.to_float(new_finished_flags) * -INF

    top_alive_seq, top_alive_log_probs, top_alive_cache = _gather_topk_beams(
        [new_seq, new_log_probs, new_cache], new_log_probs, self.batch_size,
        self.beam_size)

    return {
        _StateKeys.ALIVE_SEQ: top_alive_seq,
        _StateKeys.ALIVE_LOG_PROBS: top_alive_log_probs,
        _StateKeys.ALIVE_CACHE: top_alive_cache
    }

Response: Gather the top k sequences that are still alive and return the new state.


Instruction:
Write a Python 3 script for
combining new and old finished sequences and gather the top k sequences.

Response:
def _get_new_finished_state(self, state, new_seq, new_log_probs):
    """Combine new and old finished sequences, and gather the top k sequences.

    Args:
      state: A dictionary with the current loop state.
      new_seq: New sequences generated by growing the current alive sequences
        int32 tensor with shape [batch_size, beam_size, i + 1]
      new_log_probs: Log probabilities of new sequences
        float32 tensor with shape [batch_size, beam_size]

    Returns:
      Dictionary with finished keys from _StateKeys:
        {Top beam_size finished sequences based on score,
         Scores of finished sequences,
         Finished flags of finished sequences}
    """
    i = state[_StateKeys.CUR_INDEX]
    finished_seq = state[_StateKeys.FINISHED_SEQ]
    finished_scores = state[_StateKeys.FINISHED_SCORES]
    finished_flags = state[_StateKeys.FINISHED_FLAGS]

    # First append a column of 0-ids to finished_seq to increment the length.
    # New shape of finished_seq: [batch_size, beam_size, i + 1]
    finished_seq = tf.concat(
        [finished_seq,
         tf.zeros([self.batch_size, self.beam_size, 1], tf.int32)], axis=2)

    # Calculate new seq scores from log probabilities.
    length_norm = _length_normalization(self.alpha, i + 1)
    new_scores = new_log_probs / length_norm

    # Set the scores of the still-alive seq in new_seq to large negative values.
    new_finished_flags = tf.equal(new_seq[:, :, -1], self.eos_id)
    new_scores += (1. - tf.to_float(new_finished_flags)) * -INF

    # Combine sequences, scores, and flags.
    finished_seq = tf.concat([finished_seq, new_seq], axis=1)
    finished_scores = tf.concat([finished_scores, new_scores], axis=1)
    finished_flags = tf.concat([finished_flags, new_finished_flags], axis=1)

    # Return the finished sequences with the best scores.
    top_finished_seq, top_finished_scores, top_finished_flags = (
        _gather_topk_beams([finished_seq, finished_scores, finished_flags],
                           finished_scores, self.batch_size, self.beam_size))

    return {
        _StateKeys.FINISHED_SEQ: top_finished_seq,
        _StateKeys.FINISHED_SCORES: top_finished_scores,
        _StateKeys.FINISHED_FLAGS: top_finished_flags
    }


Instruction:
Implement a function in Python 3 to
set initial random values for trainable parameters.

Response:
def reset_parameters(self, init_weight):
        """
        Sets initial random values for trainable parameters.
        """
        stdv = 1. / math.sqrt(self.num_units)
        self.linear_att.data.uniform_(-init_weight, init_weight)

        if self.normalize:
            self.normalize_scalar.data.fill_(stdv)
            self.normalize_bias.data.zero_()


Instruction:
Can you generate a brief explanation for the following Python 3 code
def set_mask(self, context_len, context):

        if self.batch_first:
            max_len = context.size(1)
        else:
            max_len = context.size(0)

        indices = torch.arange(0, max_len, dtype=torch.int64,
                               device=context.device)
        self.mask = indices >= (context_len.unsqueeze(1))

Response:
Sets self. mask which is applied before softmax
        ones for inactive context fields zeros for active context fields


Instruction:
Can you create a Python 3 function that
calculates Bahdanau score of a set of attributes.

Response:
def calc_score(self, att_query, att_keys):
        """
        Calculate Bahdanau score

        :param att_query: b x t_q x n
        :param att_keys: b x t_k x n

        returns: b x t_q x t_k scores
        """

        b, t_k, n = att_keys.size()
        t_q = att_query.size(1)

        att_query = att_query.unsqueeze(2).expand(b, t_q, t_k, n)
        att_keys = att_keys.unsqueeze(1).expand(b, t_q, t_k, n)
        sum_qk = att_query + att_keys

        if self.normalize:
            sum_qk = sum_qk + self.normalize_bias
            linear_att = self.linear_att / self.linear_att.norm()
            linear_att = linear_att * self.normalize_scalar
        else:
            linear_att = self.linear_att

        out = torch.tanh(sum_qk).matmul(linear_att)
        return out


Instruction:
Write a Python 3 function that can
return the positional encoding.

Response:
def get_position_encoding(
    length, hidden_size, min_timescale=1.0, max_timescale=1.0e4):
  """Return positional encoding.

  Calculates the position encoding as a mix of sine and cosine functions with
  geometrically increasing wavelengths.
  Defined and formulized in Attention is All You Need, section 3.5.

  Args:
    length: Sequence length.
    hidden_size: Size of the
    min_timescale: Minimum scale that will be applied at each position
    max_timescale: Maximum scale that will be applied at each position

  Returns:
    Tensor with shape [length, hidden_size]
  """
  position = tf.to_float(tf.range(length))
  num_timescales = hidden_size // 2
  log_timescale_increment = (
      math.log(float(max_timescale) / float(min_timescale)) /
      (tf.to_float(num_timescales) - 1))
  inv_timescales = min_timescale * tf.exp(
      tf.to_float(tf.range(num_timescales)) * -log_timescale_increment)
  scaled_time = tf.expand_dims(position, 1) * tf.expand_dims(inv_timescales, 0)
  signal = tf.concat([tf.sin(scaled_time), tf.cos(scaled_time)], axis=1)
  return signal


Instruction:
Can you tell what is the following Python 3 function doing
def get_decoder_self_attention_bias(length):
  with tf.name_scope("decoder_self_attention_bias"):
    valid_locs = tf.matrix_band_part(tf.ones([length, length]), -1, 0)
    valid_locs = tf.reshape(valid_locs, [1, 1, length, length])
    decoder_bias = _NEG_INF * (1.0 - valid_locs)
  return decoder_bias

Response: Calculate bias for decoder that maintains model s autoregressive property.


Instruction:
How would you explain what the following Python 3 function does
def get_padding(x, padding_value=0):
  with tf.name_scope("padding"):
    return tf.to_float(tf.equal(x, padding_value))

Response: Returns float tensor representing the padding values in x.


Instruction:
Explain what the following Python 3 code does
def get_padding_bias(x):
  with tf.name_scope("attention_bias"):
    padding = get_padding(x)
    attention_bias = padding * _NEG_INF
    attention_bias = tf.expand_dims(
        tf.expand_dims(attention_bias, axis=1), axis=1)
  return attention_bias

Response: Calculate bias tensor from padding values in tensor.


Instruction:
Write a Python 3 function that can
add a result to the dictionary.

Response:
def _add_result(self, dict_entry, entry, dt, start_time):
    """Adds a result to the dictionary.

    Args:
      dict_entry: main dict to add entry
      entry: slot for this entry (likely an integer)
      dt: the timing for the entry
      start_time: when the entry started unix time float
    """
    time_entry = {}
    time_entry['dt'] = dt
    time_entry['start_time'] = start_time
    dict_entry[entry] = time_entry


Instruction:
Implement a function in Python 3 to
sort the results and returns an array with only the values but sorted oldest first.

Response:
def _sorted_results(self, results_dicts):
    """Sorts dict of results based on log start_time.

    Sorts the results and returns an array with only the values but sorted
    by oldest value first.value

    Args:
      results_dicts: List of result dicts

    Returns:
      List of only the time but sorted oldest first.
    """
    print('results dicts:', results_dicts)
    sorted_dict = sorted(results_dicts, key=lambda k: k['start_time'])
    results = []
    for entry in sorted_dict:
      results.append(entry['dt'])
    return results


Instruction:
Make a summary of the following Python 3 code
def get_compliance(self, filename):
    print('Running Compliance Check on {}'.format(filename))
    print('#' * 80)
    start_time, status, dt, qual, target = mlp_compliance.l2_check_file_w_starttime(
        filename)
    print('#' * 80)

    if status:
      level = '2'
    else:
      start_time, status, dt, qual, target = mlp_compliance.l1_check_file_w_starttime(
          filename)
      print('#' * 80)
      if status:
        level = '1'
      else:
        level = '0'

    success = status and qual and target and qual >= target
    return start_time, level, dt, qual, success

Response: Get the compliance level of the output file.


Instruction:
Create a Python 3 function for
verifying and result and returns timing.

Response:
def verify_and_extract_time(self, log_file, division, result_name):
    """Verifies and result and returns timing.

    Uses submodule mlp_compliance (https://github.com/bitfort/mlp_compliance)

    Args:
      log_file: Absolute path to result file.
      division: open, closed
      result_name: name of the benchmark, ncf, ssd, etc

    Returns:
      Time for the result or `INFINITE_TIME` if not a success

    Raises:
      Exception: If expected compliance level is not hit or cannot figure
      out expected compliance level.

    """
    expected_level = constants.DIVISION_COMPLIANCE_CHECK_LEVEL.get(
        division, None)
    print(result_name)
    if expected_level is None:
      raise Exception('Unknown division: {}'.format(division))
    start_time, level, dt, _, success = self.get_compliance(log_file)
    print(float(start_time))
    if int(level) != expected_level:
      raise Exception('Error Level {} does not match needed level {}:{}'.format(
          level, expected_level, log_file))

    # Sets failure to converge to "infinite time" per the rules
    if success and dt:
      return dt, start_time
    else:
      print('Result was not a success set to INFINITE_TIME({})'.format(
          INFINITE_TIME))
      return INFINITE_TIME, start_time


Instruction:
Explain what the following Python 3 code does
def _compute_and_write_row_block(
    i, left_matrix, right_matrix, train_indices_out_path, test_indices_out_path,
    remove_empty_rows):
  """Compute row block (shard) of expansion for row i of the left_matrix.

  Compute a shard of the randomized Kronecker product and dump it on the fly.
  A standard Kronecker product between matrices A and B produces
                        [[a_11 B, ..., a_1n B],
                                  ...
                         [a_m1 B, ..., a_mn B]]
    (if A's size is (m, n) and B's size is (p, q) then A Kronecker B has size
    (m p, n q)).
    Here we modify the standard Kronecker product expanding matrices in
    https://cs.stanford.edu/~jure/pubs/kronecker-jmlr10.pdf
    and randomize each block-wise operation a_ij B in the Kronecker product as
    in https://arxiv.org/pdf/1901.08910.pdf section III.4.
    The matrix we produce is
                       [[F(a_11, B, w_11), ..., F(a_1n, B, w_1n)],
                                           ...
                       [F(a_m1, B, w_m1), ... , F(a_mn, B, w_mn)]]
    where (w_ij) is a sequence of pseudo random numbers and F is randomized
    operator which will:
      1) Shuffle rows and columns of B independently at random;
      2) Dropout elements of B with a rate 1 - a_ij to compute
        F(a_ij, B, w_ij).
    (It is noteworthy that there is an abuse of notation above when writing
    F(a_ij, B, w_ij) as each block-wise operation will in fact consume
    multiple elements of the sequence (w_ij)).
  Each shard of index i consists of [F(a_i1, B, w_i1), ..., F(a_in, B, w_in)]

  Args:
    i: index of the shard. The rows i * m to (i + 1) * m of the full synthetic
      matrix matrix will be computed and dumpted to file.
    left_matrix: sparse SciPy csr matrix with values in [0, 1].
    right_matrix: sparse SciPy coo signed binary matrix. +1 values correspond
      to train set and -1 values correspond to test set.
    train_indices_out_path: path to output train file. The non zero indices of
      the resulting sparse matrix are dumped as a series of pickled records.
      '_i' will be used as a suffix for the shard's output file. The shard
      contains a pickled list of list each of which corresponds to a users.
    test_indices_out_path: path to output train file. The non zero indices of
      the resulting sparse matrix are dumped as a series of pickled records.
     '_i' will be used as a suffix for the shard's output file. The shard
      contains a pickled list of list each of which corresponds to a users.
    remove_empty_rows: whether to remove rows from the synthetic train and
      test matrices which are not present in the train or the test matrix.

  Returns:
    (num_removed_rows, metadata, train_metadata, test_metadata): an integer
      specifying the number of rows dropped because of dropout followed by
      a triplet of SparseMatrixMetadata corresponding to the overall shard,
      train shard and test shard.
  """

  kron_blocks = []

  num_rows = 0
  num_removed_rows = 0
  num_interactions = 0
  num_train_interactions = 0
  num_test_interactions = 0

  # Construct blocks
  for j in xrange(left_matrix.shape[1]):

    dropout_rate = 1.0 - left_matrix[i, j]
    kron_block = shuffle_sparse_coo_matrix(right_matrix, dropout_rate)

    if not set(kron_block.data).issubset({1, -1}):
      raise ValueError("Values of sparse matrix should be -1 or 1 but are: ",
                       set(kron_block.data))

    kron_blocks.append(kron_block)

    logging.info("Done with element (%d, %d)", i, j)

  rows_to_write = sparse.hstack(kron_blocks).tocoo()

  train_rows_to_write = util.sparse_where_equal(rows_to_write, 1)
  test_rows_to_write = util.sparse_where_equal(rows_to_write, -1)

  logging.info("Producing data set row by row")

  all_train_items_to_write = []
  all_test_items_to_write = []
  # Write Kronecker product line per line.
  for k in xrange(right_matrix.shape[0]):

    train_items_to_write = train_rows_to_write.getrow(k).indices
    test_items_to_write = test_rows_to_write.getrow(k).indices

    # for users with > 1 test items, keep only the first one
    if len(test_items_to_write) > 1:
        test_items_to_write = test_items_to_write[:1]

    num_train = train_items_to_write.shape[0]
    num_test = test_items_to_write.shape[0]

    if remove_empty_rows and ((not num_train) or (not num_test)):
      logging.info("Removed empty output row %d.",
                   i * left_matrix.shape[0] + k)
      num_removed_rows += 1
      continue

    num_rows += 1
    num_interactions += num_train + num_test
    num_train_interactions += num_train
    num_test_interactions += num_test

    all_train_items_to_write.append(train_items_to_write)
    all_test_items_to_write.append(test_items_to_write)

    if k % 1000 == 0:
      logging.info("Done producing data set row %d.", k)

  logging.info("Done producing data set row by row.")

  util.savez_two_column(
      all_train_items_to_write,
      row_offset=(i * right_matrix.shape[0]),
      file_name=train_indices_out_path + ("_%d" % i))
  util.savez_two_column(
      all_test_items_to_write,
      row_offset=(i * right_matrix.shape[0]),
      file_name=test_indices_out_path + ("_%d" % i))

  num_cols = rows_to_write.shape[1]
  metadata = SparseMatrixMetadata(num_interactions=num_interactions,
                                  num_rows=num_rows, num_cols=num_cols)
  train_metadata = SparseMatrixMetadata(num_interactions=num_train_interactions,
                                        num_rows=num_rows, num_cols=num_cols)
  test_metadata = SparseMatrixMetadata(num_interactions=num_test_interactions,
                                       num_rows=num_rows, num_cols=num_cols)

  logging.info("Done with left matrix row %d.", i)
  logging.info("%d interactions written in shard.", num_interactions)
  logging.info("%d rows removed in shard.", num_removed_rows)
  logging.info("%d train interactions written in shard.",
               num_train_interactions)
  logging.info("%d test interactions written in shard.",
               num_test_interactions)

  return (num_removed_rows, metadata, train_metadata, test_metadata)

Response: Compute and write the row block for expansion for row i of the right_matrix.


Instruction:
Write a Python 3 function that can
compute randomized Kronecker product and dump it on the fly. A standard Kronecker product between matrices A and B produces [[a_11 B, ..., a_1n B], ... [a_m1 B, ..., a_mn B]] (if A's size is (m, n) and B's size is (p, q) then A Kronecker B has size (m p, n q)). Here we modify the standard Kronecker product expanding matrices in https://cs.stanford.edu/~jure/pubs/kronecker-jmlr10.pdf and randomize each block-wise operation a_ij B in the Kronecker product as in https://arxiv.org/pdf/1901.08910.pdf section III.4. The matrix we produce is [[F(a_11, B, w_11), ..., F(a_1n, B, w_1n)], ... [F(a_m1, B, w_m1), ... , F(a_mn, B, w_mn)]] where (w_ij) is a sequence of pseudo random numbers and F is randomized operator which will: 1) Shuffle rows and columns of B independently at random; 2) Dropout elements of B with a rate 1 - a_ij to compute F(a_ij, B, w_ij). (It is noteworthy that there is an abuse of notation above when writing F(a_ij, B, w_ij) as each block-wise operation will in fact consume multiple elements of the sequence (w_ij)). Args: left_matrix: sparse SciPy csr matrix with values in [0, 1]. right_matrix: sparse SciPy coo signed binary matrix. +1 values correspond to train set and -1 values correspond to test set. train_indices_out_path: path to output train file. The non zero indices of the resulting sparse matrix are dumped as a series of pickled records. As many shard will be created as there are rows in left matrix. The shard corresponding to row i in the left matrix has the suffix _i appended to its file name. Each shard contains a pickled list of list each of which corresponds to a users. test_indices_out_path: path to output train file. The non zero indices of the resulting sparse matrix are dumped as a series of pickled records. As many shard will be created as there are rows in left matrix. The shard corresponding to row i in the left matrix has the suffix _i appended to its file name. Each shard contains a pickled list of list each of which corresponds to a users. train_metadata_out_path: path to optional complementary output file containing the number of train rows (r), columns (c) and non zeros (nnz) in a pickled SparseMatrixMetadata named tuple. test_metadata_out_path: path to optional complementary output file containing the number of test rows (r), columns (c) and non zeros (nnz) in a pickled SparseMatrixMetadata named tuple. remove_empty_rows: whether to remove rows from the synthetic train and test matrices which are not present in the train or the test matrix. Returns: (metadata, train_metadata, test_metadata) triplet of SparseMatrixMetadata corresponding to the overall data set, train data set and test data set.

Response:
def output_randomized_kronecker_to_pickle(
    left_matrix, right_matrix,
    train_indices_out_path, test_indices_out_path,
    train_metadata_out_path=None, test_metadata_out_path=None,
    remove_empty_rows=True):
  """Compute randomized Kronecker product and dump it on the fly.

  A standard Kronecker product between matrices A and B produces
                        [[a_11 B, ..., a_1n B],
                                  ...
                         [a_m1 B, ..., a_mn B]]
    (if A's size is (m, n) and B's size is (p, q) then A Kronecker B has size
    (m p, n q)).
    Here we modify the standard Kronecker product expanding matrices in
    https://cs.stanford.edu/~jure/pubs/kronecker-jmlr10.pdf
    and randomize each block-wise operation a_ij B in the Kronecker product as
    in https://arxiv.org/pdf/1901.08910.pdf section III.4.
    The matrix we produce is
                       [[F(a_11, B, w_11), ..., F(a_1n, B, w_1n)],
                                           ...
                       [F(a_m1, B, w_m1), ... , F(a_mn, B, w_mn)]]
    where (w_ij) is a sequence of pseudo random numbers and F is randomized
    operator which will:
      1) Shuffle rows and columns of B independently at random;
      2) Dropout elements of B with a rate 1 - a_ij to compute
        F(a_ij, B, w_ij).
    (It is noteworthy that there is an abuse of notation above when writing
    F(a_ij, B, w_ij) as each block-wise operation will in fact consume
    multiple elements of the sequence (w_ij)).

  Args:
    left_matrix: sparse SciPy csr matrix with values in [0, 1].
    right_matrix: sparse SciPy coo signed binary matrix. +1 values correspond
      to train set and -1 values correspond to test set.
    train_indices_out_path: path to output train file. The non zero indices of
      the resulting sparse matrix are dumped as a series of pickled records.
      As many shard will be created as there are rows in left matrix. The shard
      corresponding to row i in the left matrix has the suffix _i appended to
      its file name. Each shard contains a pickled list of list each of which
      corresponds to a users.
    test_indices_out_path: path to output train file. The non zero indices of
      the resulting sparse matrix are dumped as a series of pickled records.
      As many shard will be created as there are rows in left matrix. The shard
      corresponding to row i in the left matrix has the suffix _i appended to
      its file name. Each shard contains a pickled list of list each of which
      corresponds to a users.
    train_metadata_out_path: path to optional complementary output file
      containing the number of train rows (r), columns (c) and non zeros (nnz)
      in a pickled SparseMatrixMetadata named tuple.
    test_metadata_out_path: path to optional complementary output file
      containing the number of test rows (r), columns (c) and non zeros (nnz)
      in a pickled SparseMatrixMetadata named tuple.
    remove_empty_rows: whether to remove rows from the synthetic train and
      test matrices which are not present in the train or the test matrix.

  Returns:
    (metadata, train_metadata, test_metadata) triplet of SparseMatrixMetadata
      corresponding to the overall data set, train data set and test data set.
  """
  logging.info("Writing item sequences to pickle files %s and %s.",
               train_indices_out_path, test_indices_out_path)

  num_rows = 0
  num_removed_rows = 0
  num_cols = left_matrix.shape[1] * right_matrix.shape[1]
  num_interactions = 0

  num_train_interactions = 0
  num_test_interactions = 0

  if not set(right_matrix.data).issubset({-1, 1}):
    raise ValueError(
        "Values of sparse matrix should be -1 or 1 but are:",
        set(right_matrix.data))

  for i in xrange(left_matrix.shape[0]):

    (shard_num_removed_rows, shard_metadata, shard_train_metadata,
     shard_test_metadata) = _compute_and_write_row_block(
         i, left_matrix, right_matrix, train_indices_out_path,
         test_indices_out_path, remove_empty_rows)

    num_rows += shard_metadata.num_rows
    num_removed_rows += shard_num_removed_rows
    num_interactions += shard_metadata.num_interactions
    num_train_interactions += shard_train_metadata.num_interactions
    num_test_interactions += shard_test_metadata.num_interactions

    logging.info("%d total interactions written.", num_interactions)
    logging.info("%d total rows removed.", num_removed_rows)
    logging.info("%d total train interactions written.", num_train_interactions)
    logging.info("%d toal test interactions written.", num_test_interactions)

  logging.info("Done writing.")

  metadata = SparseMatrixMetadata(
      num_interactions=num_interactions,
      num_rows=num_rows, num_cols=num_cols)
  train_metadata = SparseMatrixMetadata(
      num_interactions=num_train_interactions,
      num_rows=num_rows, num_cols=num_cols)
  test_metadata = SparseMatrixMetadata(
      num_interactions=num_test_interactions,
      num_rows=num_rows, num_cols=num_cols)

  if train_metadata_out_path is not None:
    util.write_metadata_to_file(
        train_metadata, train_metadata_out_path, tag="train")
  if test_metadata_out_path is not None:
    util.write_metadata_to_file(
        test_metadata, test_metadata_out_path, tag="test")

  return metadata, train_metadata, test_metadata


Instruction:
How would you explain what the following Python 3 function does
def make_gtp_instance(load_file, cgos_mode=False, kgs_mode=False,
                      minigui_mode=False):
    """Takes a path to model files and set up a GTP engine instance."""
    n = DualNetwork(load_file)
    if cgos_mode:
        player = CGOSPlayer(network=n, seconds_per_move=5, timed_match=True,
                            two_player_mode=True)
    else:
        player = MCTSPlayer(network=n, two_player_mode=True)

    name = "Minigo-" + os.path.basename(load_file)
    version = "0.2"

    engine = gtp_engine.Engine()
    engine.add_cmd_handler(
        gtp_engine.EngineCmdHandler(engine, name, version))

    if kgs_mode:
        engine.add_cmd_handler(KgsCmdHandler(player))
    engine.add_cmd_handler(RegressionsCmdHandler(player))
    engine.add_cmd_handler(GoGuiCmdHandler(player))
    if minigui_mode:
        engine.add_cmd_handler(MiniguiBasicCmdHandler(player, courtesy_pass=kgs_mode))
    else:
        engine.add_cmd_handler(BasicCmdHandler(player, courtesy_pass=kgs_mode))

    return engine

Response: Takes a path to model files and sets up a GTP engine instance.


Instruction:
Can you generate the documentation for the following Python 3 function
def main(argv):
    del argv
    engine = make_gtp_instance(FLAGS.load_file,
                               cgos_mode=FLAGS.cgos_mode,
                               kgs_mode=FLAGS.kgs_mode,
                               minigui_mode=FLAGS.minigui_mode)
    dbg("GTP engine ready\n")
    for msg in sys.stdin:
        if not engine.handle_msg(msg.strip()):
            break

Response: Run Minigo in GTP mode.


Instruction:
Explain what the following Python 3 code does
def validate(*tf_records):
    if FLAGS.use_tpu:
        def _input_fn(params):
            return preprocessing.get_tpu_input_tensors(
                params['batch_size'], tf_records, filter_amount=1.0)
    else:
        def _input_fn():
            return preprocessing.get_input_tensors(
                FLAGS.train_batch_size, tf_records, filter_amount=1.0,
                shuffle_examples=False)

    steps = FLAGS.examples_to_validate // FLAGS.train_batch_size
    if FLAGS.use_tpu:
        steps //= FLAGS.num_tpu_cores

    estimator = dual_net.get_estimator()
    with utils.logged_timer("Validating"):
        estimator.evaluate(_input_fn, steps=steps, name=FLAGS.validate_name)

Response: Validate a model s performance on a set of holdout data.


Instruction:
Make a summary of the following Python 3 code
def main(argv):
    _, *validation_paths = argv
    if FLAGS.expand_validation_dirs:
        tf_records = []
        with utils.logged_timer("Building lists of holdout files"):
            for record_dir in validation_paths:
                tf_records.extend(gfile.Glob(os.path.join(record_dir, '*.zz')))
    else:
        tf_records = validation_paths

    if not tf_records:
        raise RuntimeError("Did not find any holdout files for validating!")
    validate(*tf_records)

Response: Validate a model s performance on a set of holdout data.


Instruction:
Can you write a function in Python 3 where it
loads custom environment setup from a Python source file and run the setup function.

Response:
def setup_custom_environment(custom_module_path):
    """Load custom environment setup from a Python source file and run the setup
    function.
    """
    module = import_file("maskrcnn_benchmark.utils.env.custom_module", custom_module_path)
    assert hasattr(module, "setup_environment") and callable(
        module.setup_environment
    ), (
        "Custom environment module defined in {} does not have the "
        "required callable attribute 'setup_environment'."
    ).format(
        custom_module_path
    )
    module.setup_environment()


Instruction:
How would you implement a function in Python 3 that
finds all models returning a list of model number and names sorted increasing.

Response:
def get_models():
    """Finds all models, returning a list of model number and names
    sorted increasing.

    Returns: [(13, 000013-modelname), (17, 000017-modelname), ...etc]
    """
    all_models = gfile.Glob(os.path.join(models_dir(), '*.meta'))
    model_filenames = [os.path.basename(m) for m in all_models]
    model_numbers_names = sorted([
        (shipname.detect_model_num(m), shipname.detect_model_name(m))
        for m in model_filenames])
    return model_numbers_names


Instruction:
Can you generate the documentation for the following Python 3 function
def get_hour_dirs(root=None):
    root = root or selfplay_dir()
    return list(filter(lambda s: re.match(r"\d{4}-\d{2}-\d{2}-\d{2}", s),
                       gfile.ListDirectory(root)))

Response: Gets the directories under selfplay_dir that match YYYY - MM - DD - HH.


Instruction:
Can you create a Python 3 function that
prints statistics for the most recent n_back models

Response:
def game_counts(n_back=20):
    """Prints statistics for the most recent n_back models"""
    for _, model_name in get_models[-n_back:]:
        games = get_games(model_name)
        print("Model: {}, Games: {}".format(model_name, len(games)))


Instruction:
Write a Python 3 script to
perform non - maximum suppression on a boxlist.

Response:
def boxlist_nms(boxlist, nms_thresh, max_proposals=-1, score_field="scores"):
    """
    Performs non-maximum suppression on a boxlist, with scores specified
    in a boxlist field via score_field.

    Arguments:
        boxlist(BoxList)
        nms_thresh (float)
        max_proposals (int): if > 0, then only the top max_proposals are kept
            after non-maximum suppression
        score_field (str)
    """
    if nms_thresh <= 0:
        return boxlist
    mode = boxlist.mode
    boxlist = boxlist.convert("xyxy")
    boxes = boxlist.bbox
    score = boxlist.get_field(score_field)
    keep = _box_nms(boxes, score, nms_thresh)
    if max_proposals > 0:
        keep = keep[: max_proposals]
    boxlist = boxlist[keep]
    return boxlist.convert(mode)


Instruction:
Can you create a Python 3 function that
removes boxes with both sides >= min_size

Response:
def remove_small_boxes(boxlist, min_size):
    """
    Only keep boxes with both sides >= min_size

    Arguments:
        boxlist (Boxlist)
        min_size (int)
    """
    # TODO maybe add an API for querying the ws / hs
    xywh_boxes = boxlist.convert("xywh").bbox
    _, _, ws, hs = xywh_boxes.unbind(dim=1)
    keep = (
        (ws >= min_size) & (hs >= min_size)
    ).nonzero().squeeze(1)
    return boxlist[keep]


Instruction:
Can you tell what is the following Python 3 function doing
def boxlist_iou(boxlist1, boxlist2):
    if boxlist1.size != boxlist2.size:
        raise RuntimeError(
                "boxlists should have same image size, got {}, {}".format(boxlist1, boxlist2))

    N = len(boxlist1)
    M = len(boxlist2)

    area1 = boxlist1.area()
    area2 = boxlist2.area()

    box1, box2 = boxlist1.bbox, boxlist2.bbox

    lt = torch.max(box1[:, None, :2], box2[:, :2])  # [N,M,2]
    rb = torch.min(box1[:, None, 2:], box2[:, 2:])  # [N,M,2]

    TO_REMOVE = 1

    wh = (rb - lt + TO_REMOVE).clamp(min=0)  # [N,M,2]
    inter = wh[:, :, 0] * wh[:, :, 1]  # [N,M]

    iou = inter / (area1[:, None] + area2 - inter)
    return iou

Response: Compute the intersection over two set of boxes.


Instruction:
Write a Python 3 function that can
concatenate a list of BoxList into a

Response:
def cat_boxlist(bboxes):
    """
    Concatenates a list of BoxList (having the same image size) into a
    single BoxList

    Arguments:
        bboxes (list[BoxList])
    """
    assert isinstance(bboxes, (list, tuple))
    assert all(isinstance(bbox, BoxList) for bbox in bboxes)

    size = bboxes[0].size
    assert all(bbox.size == size for bbox in bboxes)

    mode = bboxes[0].mode
    assert all(bbox.mode == mode for bbox in bboxes)

    fields = set(bboxes[0].fields())
    assert all(set(bbox.fields()) == fields for bbox in bboxes)

    cat_boxes = BoxList(_cat([bbox.bbox for bbox in bboxes], dim=0), size, mode)

    for field in fields:
        data = _cat([bbox.get_field(field) for bbox in bboxes], dim=0)
        cat_boxes.add_field(field, data)

    return cat_boxes


Instruction:
Here you have a function in Python 3, explain what it does
def forward(self, ploc, plabel, gloc, glabel):

        mask = glabel > 0
        pos_num = mask.sum(dim=1)

        vec_gd = self._loc_vec(gloc)

        # sum on four coordinates, and mask
        sl1 = self.sl1_loss(ploc, vec_gd).sum(dim=1)
        sl1 = (mask.float()*sl1).sum(dim=1)

        # hard negative mining
        con = self.con_loss(plabel, glabel)

        # postive mask will never selected
        con_neg = con.clone()
        con_neg[mask] = 0
        _, con_idx = con_neg.sort(dim=1, descending=True)
        _, con_rank = con_idx.sort(dim=1)

        # number of negative three times positive
        neg_num = torch.clamp(3*pos_num, max=mask.size(1)).unsqueeze(-1)
        neg_mask = con_rank < neg_num

        closs = (con*(mask.float() + neg_mask.float())).sum(dim=1)

        # avoid no object detected
        total_loss = sl1 + closs
        num_mask = (pos_num > 0).float()
        pos_num = pos_num.float().clamp(min=1e-6)

        ret = (total_loss*num_mask/pos_num).mean(dim=0)
        return ret

Response: forward method for the internal method of the forward method.


Instruction:
Write a Python 3 function that can
set gradients of the parameters to the gradients of the parameters_with_grad.

Response:
def set_grads(params, params_with_grad):
        """
        Copies gradients from param_with_grad to params

        :param params: dst parameters
        :param params_with_grad: src parameters
        """
        for param, param_w_grad in zip(params, params_with_grad):
            if param.grad is None:
                param.grad = torch.nn.Parameter(torch.empty_like(param))
            param.grad.data.copy_(param_w_grad.grad.data)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def set_weights(params, new_params):
        for param, new_param in zip(params, new_params):
            param.data.copy_(new_param.data)

Response: Sets the weights of the parameters in the new_params to the ones in the new_params.


Instruction:
Can you generate the documentation for the following Python 3 function
def initialize_model(self, model):
        logging.info('Initializing fp32 clone weights')
        self.fp16_model = model
        self.fp16_model.zero_grad()
        self.fp32_params = [param.to(torch.float32).detach()
                            for param in model.parameters()]

        for param in self.fp32_params:
            param.requires_grad = True

Response: Initializes internal state and build fp32 master copy of weights.


Instruction:
How would you explain what the following Python 3 function does
def step(self, loss, optimizer, scheduler, update=True):
        loss *= self.loss_scale
        loss.backward()

        if update:
            self.set_grads(self.fp32_params, self.fp16_model.parameters())
            if self.loss_scale != 1.0:
                for param in self.fp32_params:
                    param.grad.data /= self.loss_scale

            norm = clip_grad_norm_(self.fp32_params, self.grad_clip)

            if math.isfinite(norm):
                scheduler.step()
                optimizer.step()
                self.set_weights(self.fp16_model.parameters(),
                                 self.fp32_params)
                self.since_last_invalid += 1
            else:
                self.loss_scale /= self.dls_downscale
                self.since_last_invalid = 0
                logging.info(f'Gradient norm: {norm}')
                logging.info(f'Skipped batch, new scale: {self.loss_scale}')

            if self.since_last_invalid >= self.dls_upscale_interval:
                self.loss_scale *= self.dls_upscale
                self.loss_scale = min(self.loss_scale, 8192.0)
                logging.info(f'Upscaling, new scale: {self.loss_scale}')
                self.since_last_invalid = 0

            self.fp16_model.zero_grad()

Response: Performs one step of the optimizer.


Instruction:
Can you implement a function in Python 3 that
performs one step of the optimizer.

Response:
def step(self, loss, optimizer, scheduler, update=True):
        """
        Performs one step of the optimizer.

        :param loss: value of loss function
        :param optimizer: optimizer
        :param update: if True executes weight update
        """
        loss.backward()
        if update:
            if self.grad_clip != float('inf'):
                clip_grad_norm_(self.model.parameters(), self.grad_clip)
            scheduler.step()
            optimizer.step()
            self.model.zero_grad()


Instruction:
Create a Python 3 function to
return inputs and targets Tensors from a serialized tf. Example.

Response:
def _parse_example(serialized_example):
  """Return inputs and targets Tensors from a serialized tf.Example."""
  data_fields = {
      "inputs": tf.VarLenFeature(tf.int64),
      "targets": tf.VarLenFeature(tf.int64)
  }
  parsed = tf.parse_single_example(serialized_example, data_fields)
  inputs = tf.sparse_tensor_to_dense(parsed["inputs"])
  targets = tf.sparse_tensor_to_dense(parsed["targets"])
  return inputs, targets


Instruction:
How would you implement a function in Python 3 that
indicates whether the example s length is lower than the maximum length.

Response:
def _filter_max_length(example, max_length=256):
  """Indicates whether the example's length is lower than the maximum length."""
  return tf.logical_and(tf.size(example[0]) <= max_length,
                        tf.size(example[1]) <= max_length)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _get_example_length(example):
  length = tf.maximum(tf.shape(example[0])[0], tf.shape(example[1])[0])
  return length

Response: Returns the maximum length between the example inputs and targets.


Instruction:
Write a Python 3 script to
create min and max boundary lists.

Response:
def _create_min_max_boundaries(
    max_length, min_boundary=_MIN_BOUNDARY, boundary_scale=_BOUNDARY_SCALE):
  """Create min and max boundary lists up to max_length.

  For example, when max_length=24, min_boundary=4 and boundary_scale=2, the
  returned values will be:
    buckets_min = [0, 4, 8, 16, 24]
    buckets_max = [4, 8, 16, 24, 25]

  Args:
    max_length: The maximum length of example in dataset.
    min_boundary: Minimum length in boundary.
    boundary_scale: Amount to scale consecutive boundaries in the list.

  Returns:
    min and max boundary lists

  """
  # Create bucket boundaries list by scaling the previous boundary or adding 1
  # (to ensure increasing boundary sizes).
  bucket_boundaries = []
  x = min_boundary
  while x < max_length:
    bucket_boundaries.append(x)
    x = max(x + 1, int(x * boundary_scale))

  # Create min and max boundary lists from the initial list.
  buckets_min = [0] + bucket_boundaries
  buckets_max = bucket_boundaries + [max_length + 1]
  return buckets_min, buckets_max


Instruction:
Can you generate the documentation for the following Python 3 function
def _batch_examples(dataset, batch_size, max_length):
  # Get min and max boundary lists for each example. These are used to calculate
  # the `bucket_id`, which is the index at which:
  # buckets_min[bucket_id] <= len(example) < buckets_max[bucket_id]
  # Note that using both min and max lists improves the performance.
  buckets_min, buckets_max = _create_min_max_boundaries(max_length)

  # Create list of batch sizes for each bucket_id, so that
  # bucket_batch_size[bucket_id] * buckets_max[bucket_id] <= batch_size
  bucket_batch_sizes = [batch_size // x for x in buckets_max]
  # bucket_id will be a tensor, so convert this list to a tensor as well.
  bucket_batch_sizes = tf.constant(bucket_batch_sizes, dtype=tf.int64)

  def example_to_bucket_id(example_input, example_target):
    """Return int64 bucket id for this example, calculated based on length."""
    seq_length = _get_example_length((example_input, example_target))

    # TODO: investigate whether removing code branching improves performance.
    conditions_c = tf.logical_and(
        tf.less_equal(buckets_min, seq_length),
        tf.less(seq_length, buckets_max))
    bucket_id = tf.reduce_min(tf.where(conditions_c))
    return bucket_id

  def window_size_fn(bucket_id):
    """Return number of examples to be grouped when given a bucket id."""
    return bucket_batch_sizes[bucket_id]

  def batching_fn(bucket_id, grouped_dataset):
    """Batch and add padding to a dataset of elements with similar lengths."""
    bucket_batch_size = window_size_fn(bucket_id)

    # Batch the dataset and add padding so that all input sequences in the
    # examples have the same length, and all target sequences have the same
    # lengths as well. Resulting lengths of inputs and targets can differ.
    return grouped_dataset.padded_batch(bucket_batch_size, ([None], [None]))

  return dataset.apply(tf.contrib.data.group_by_window(
      key_func=example_to_bucket_id,
      reduce_func=batching_fn,
      window_size=None,
      window_size_func=window_size_fn))

Response: Group examples by similar lengths and return batched dataset.


Instruction:
Write a Python 3 script to
read and batch from files.

Response:
def _read_and_batch_from_files(
    file_pattern, batch_size, max_length, num_cpu_cores, shuffle, repeat):
  """Create dataset where each item is a dict of "inputs" and "targets".

  Args:
    file_pattern: String used to match the input TFRecord files.
    batch_size: Maximum number of tokens per batch of examples
    max_length: Maximum number of tokens per example
    num_cpu_cores: Number of cpu cores for parallel input processing.
    shuffle: If true, randomizes order of elements.
    repeat: Number of times to repeat the dataset. If None, the dataset is
      repeated forever.

  Returns:
    tf.data.Dataset object containing examples loaded from the files.
  """
  dataset = tf.data.Dataset.list_files(file_pattern)

  if shuffle:
    # Shuffle filenames
    mlperf_log.transformer_print(key=mlperf_log.INPUT_ORDER)
    dataset = dataset.shuffle(buffer_size=_FILE_SHUFFLE_BUFFER)

  # Read files and interleave results. When training, the order of the examples
  # will be non-deterministic.
  dataset = dataset.apply(
      tf.contrib.data.parallel_interleave(
          _load_records, sloppy=shuffle, cycle_length=num_cpu_cores))

  # Parse each tf.Example into a dictionary
  # TODO: Look into prefetch_input_elements for performance optimization.
  dataset = dataset.map(_parse_example,
                        num_parallel_calls=num_cpu_cores)

  # Remove examples where the input or target length exceeds the maximum length,
  dataset = dataset.filter(lambda x, y: _filter_max_length((x, y), max_length))

  # Batch such that each batch has examples of similar length.
  mlperf_log.transformer_print(key=mlperf_log.INPUT_BATCH_SIZE,
                               value=batch_size)
  mlperf_log.transformer_print(key=mlperf_log.INPUT_MAX_LENGTH,
                               value=max_length)
  dataset = _batch_examples(dataset, batch_size, max_length)
  dataset = dataset.repeat(repeat)

  # Prefetch the next element to improve speed of input pipeline.
  dataset = dataset.prefetch(1)
  return dataset


Instruction:
Write a Python 3 script to
load and return dataset of batched examples for use during training.

Response:
def train_input_fn(params):
  """Load and return dataset of batched examples for use during training."""
  file_pattern = os.path.join(getattr(params, "data_dir", ""), "*encoded-train*")
  return _read_and_batch_from_files(
      file_pattern, params.batch_size, params.max_length, params.num_cpu_cores,
      shuffle=True, repeat=params.repeat_dataset)


Instruction:
Write a Python 3 function for
loading and return dataset of batched examples for use during evaluation.

Response:
def eval_input_fn(params):
  """Load and return dataset of batched examples for use during evaluation."""
  file_pattern = os.path.join(getattr(params, "data_dir", ""), "*encoded-dev*")
  return _read_and_batch_from_files(
      file_pattern, params.batch_size, params.max_length, params.num_cpu_cores,
      shuffle=False, repeat=1)


Instruction:
Can you generate the documentation for the following Python 3 function
def reduce_loss_dict(loss_dict):
    world_size = get_world_size()
    if world_size < 2:
        return loss_dict
    with torch.no_grad():
        loss_names = []
        all_losses = []
        for k in sorted(loss_dict.keys()):
            loss_names.append(k)
            all_losses.append(loss_dict[k])
        all_losses = torch.stack(all_losses, dim=0)
        dist.reduce(all_losses, dst=0)
        if dist.get_rank() == 0:
            # only main process gets accumulated, so only divide by
            # world_size in this case
            all_losses /= world_size
        reduced_losses = {k: v for k, v in zip(loss_names, all_losses)}
    return reduced_losses

Response:
Reduce the loss dictionary from all processes so that process with rank
    0 has the averaged results. Returns a dict with the same fields as loss_dict.


Instruction:
Implement a Python 3 function for
converting dtype string to tf dtype and set loss_scale default as needed.

Response:
def parse_dtype_info(flags):
  """Convert dtype string to tf dtype, and set loss_scale default as needed.

  Args:
    flags: namespace object returned by arg parser.

  Raises:
    ValueError: If an invalid dtype is provided.
  """
  if flags.dtype in (i[0] for i in DTYPE_MAP.values()):
    return  # Make function idempotent

  try:
    flags.dtype, default_loss_scale = DTYPE_MAP[flags.dtype]
  except KeyError:
    raise ValueError("Invalid dtype: {}".format(flags.dtype))

  flags.loss_scale = flags.loss_scale or default_loss_scale


Instruction:
Write a Python 3 script to
calculate the IoU tensor based on two boxes tensor.

Response:
def calc_iou_tensor(box1, box2):
    """ Calculation of IoU based on two boxes tensor,
        Reference to https://github.com/kuangliu/pytorch-ssd
        input:
            box1 (N, 4)
            box2 (M, 4)
        output:
            IoU (N, M)
    """
    N = box1.size(0)
    M = box2.size(0)

    be1 = box1.unsqueeze(1).expand(-1, M, -1)
    be2 = box2.unsqueeze(0).expand(N, -1, -1)

    # Left Top & Right Bottom
    lt = torch.max(be1[:, :, :2], be2[:, :, :2])
    # mask1 = (be1[:,:, 0] < be2[:,:, 0]) ^ (be1[:,:, 1] < be2[:,:, 1])
    # mask1 = ~mask1
    rb = torch.min(be1[:, :, 2:], be2[:, :, 2:])
    # mask2 = (be1[:,:, 2] < be2[:,:, 2]) ^ (be1[:,:, 3] < be2[:,:, 3])
    # mask2 = ~mask2

    delta = rb - lt
    delta[delta < 0] = 0
    intersect = delta[:, :, 0] * delta[:, :, 1]
    # *mask1.float()*mask2.float()

    delta1 = be1[:, :, 2:] - be1[:, :, :2]
    area1 = delta1[:, :, 0] * delta1[:, :, 1]
    delta2 = be2[:, :, 2:] - be2[:, :, :2]
    area2 = delta2[:, :, 0] * delta2[:, :, 1]

    iou = intersect / (area1 + area2 - intersect)
    return iou


Instruction:
Create a Python 3 function for
scaling and transform from xywh to ltrb

Response:
def scale_back_batch(self, bboxes_in, scores_in):
        """
            Do scale and transform from xywh to ltrb
            suppose input Nx4xnum_bbox Nxlabel_numxnum_bbox
        """
        if bboxes_in.device == torch.device("cpu"):
            self.dboxes = self.dboxes.cpu()
            self.dboxes_xywh = self.dboxes_xywh.cpu()
        else:
            self.dboxes = self.dboxes.cuda()
            self.dboxes_xywh = self.dboxes_xywh.cuda()

        bboxes_in = bboxes_in.permute(0, 2, 1)
        scores_in = scores_in.permute(0, 2, 1)
        # print(bboxes_in.device, scores_in.device, self.dboxes_xywh.device)

        bboxes_in[:, :, :2] = self.scale_xy * bboxes_in[:, :, :2]
        bboxes_in[:, :, 2:] = self.scale_wh * bboxes_in[:, :, 2:]

        bboxes_in[:, :, :2] = bboxes_in[:, :, :2] * self.dboxes_xywh[:, :,
                                                    2:] + self.dboxes_xywh[:, :,
                                                          :2]
        bboxes_in[:, :, 2:] = bboxes_in[:, :, 2:].exp() * self.dboxes_xywh[:, :,
                                                          2:]

        # Transform format to ltrb
        l, t, r, b = bboxes_in[:, :, 0] - 0.5 * bboxes_in[:, :, 2], \
                     bboxes_in[:, :, 1] - 0.5 * bboxes_in[:, :, 3], \
                     bboxes_in[:, :, 0] + 0.5 * bboxes_in[:, :, 2], \
                     bboxes_in[:, :, 1] + 0.5 * bboxes_in[:, :, 3]

        bboxes_in[:, :, 0] = l
        bboxes_in[:, :, 1] = t
        bboxes_in[:, :, 2] = r
        bboxes_in[:, :, 3] = b

        return bboxes_in, F.softmax(scores_in, dim=-1)


Instruction:
Can you tell what is the following Python 3 function doing
def generate_anchors(
    stride=16, sizes=(32, 64, 128, 256, 512), aspect_ratios=(0.5, 1, 2)
):
    """Generates a matrix of anchor boxes in (x1, y1, x2, y2) format. Anchors
    are centered on stride / 2, have (approximate) sqrt areas of the specified
    sizes, and aspect ratios as given.
    """
    return _generate_anchors(
        stride,
        np.array(sizes, dtype=np.float) / stride,
        np.array(aspect_ratios, dtype=np.float),
    )

Response: Generates a matrix of anchor boxes in the specified size and aspect ratios.


Instruction:
Can you tell what is the following Python 3 function doing
def _generate_anchors(base_size, scales, aspect_ratios):
    anchor = np.array([1, 1, base_size, base_size], dtype=np.float) - 1
    anchors = _ratio_enum(anchor, aspect_ratios)
    anchors = np.vstack(
        [_scale_enum(anchors[i, :], scales) for i in range(anchors.shape[0])]
    )
    return torch.from_numpy(anchors)

Response:
Generate anchor windows by enumerating aspect ratios X
    scales wrt a reference


Instruction:
Can you create a Python 3 function that
converts from a flattened coordinate to a Minigo coordinate.

Response:
def from_flat(flat):
    """Converts from a flattened coordinate to a Minigo coordinate."""
    if flat == go.N * go.N:
        return None
    return divmod(flat, go.N)


Instruction:
How would you explain what the following Python 3 function does
def to_flat(coord):
    if coord is None:
        return go.N * go.N
    return go.N * coord[0] + coord[1]

Response: Converts from a Minigo coordinate to a flattened coordinate.


Instruction:
Here you have a function in Python 3, explain what it does
def from_sgf(sgfc):
    if sgfc is None or sgfc == '' or (go.N <= 19 and sgfc == 'tt'):
        return None
    return _SGF_COLUMNS.index(sgfc[1]), _SGF_COLUMNS.index(sgfc[0])

Response: Converts from an SGF coordinate to a Minigo coordinate.


Instruction:
Can you generate the documentation for the following Python 3 function
def from_gtp(gtpc):
    gtpc = gtpc.upper()
    if gtpc == 'PASS':
        return None
    col = _GTP_COLUMNS.index(gtpc[0])
    row_from_bottom = int(gtpc[1:])
    return go.N - row_from_bottom, col

Response: Converts from a GTP coordinate to a Minigo coordinate.


Instruction:
Create a Python 3 function for
converting from a Minigo coordinate to a GTP coordinate.

Response:
def to_gtp(coord):
    """Converts from a Minigo coordinate to a GTP coordinate."""
    if coord is None:
        return 'pass'
    y, x = coord
    return '{}{}'.format(_GTP_COLUMNS[x], go.N - y)


Instruction:
Create a Python 3 function for
wrapping for cv2. findContours to maintain compatiblity between versions 3 and 4

Response:
def findContours(*args, **kwargs):
    """
    Wraps cv2.findContours to maintain compatiblity between versions
    3 and 4

    Returns:
        contours, hierarchy
    """
    if cv2.__version__.startswith('4'):
        contours, hierarchy = cv2.findContours(*args, **kwargs)
    elif cv2.__version__.startswith('3'):
        _, contours, hierarchy = cv2.findContours(*args, **kwargs)
    else:
        raise AssertionError(
            'cv2 must be either version 3 or 4 to call this method')

    return contours, hierarchy


Instruction:
How would you explain what the following Python 3 function does
def maybe_add_child(self, fcoord):
        if fcoord not in self.children:
            new_position = self.position.play_move(
                coords.from_flat(fcoord))
            self.children[fcoord] = MCTSNode(
                new_position, fmove=fcoord, parent=self)
        return self.children[fcoord]

Response: Adds a child node for fcoord if it doesn t already exist.


Instruction:
Can you tell what is the following Python 3 function doing
def add_virtual_loss(self, up_to):
        self.losses_applied += 1
        # This is a "win" for the current node; hence a loss for its parent node
        # who will be deciding whether to investigate this node again.
        loss = self.position.to_play
        self.W += loss
        if self.parent is None or self is up_to:
            return
        self.parent.add_virtual_loss(up_to)

Response: Propagate a virtual loss up to the root node.


Instruction:
Write a Python 3 function that can
propagate a value estimation up to the root node.

Response:
def backup_value(self, value, up_to):
        """Propagates a value estimation up to the root node.

        Args:
            value: the value to be propagated (1 = black wins, -1 = white wins)
            up_to: the node to propagate until.
        """
        self.N += 1
        self.W += value
        if self.parent is None or self is up_to:
            return
        self.parent.backup_value(value, up_to)


Instruction:
Here you have a function in Python 3, explain what it does
def is_done(self):
        return self.position.is_game_over() or self.position.n >= FLAGS.max_game_length

Response: True if the last two moves were Pass or if the position is at a game over or greater than the max depth.


Instruction:
Given the following Python 3 function, write the documentation
def children_as_pi(self, squash=False):
        probs = self.child_N
        if squash:
            probs = probs ** .98
        sum_probs = np.sum(probs)
        if sum_probs == 0:
            return probs
        return probs / np.sum(probs)

Response: Returns the child visit counts as a probability distribution


Instruction:
Create a Python 3 function for
returning the most visited path in go - gui VAR format e. g. b r3 w c17...

Response:
def mvp_gg(self):
        """ Returns most visited path in go-gui VAR format e.g. 'b r3 w c17..."""
        output = []
        for node in self.most_visited_path_nodes():
            if max(node.child_N) <= 1:
                break
            output.append(coords.to_gtp(coords.from_flat(node.fmove)))
        return ' '.join(output)


Instruction:
Explain what the following Python 3 code does
def _convert_args(handler, args):

    args = list(args)
    params = inspect.signature(handler).parameters
    for i, (arg, name) in enumerate(zip(args, params)):
        default = params[name].default
        annotation = params[name].annotation

        if annotation != inspect.Parameter.empty:
            if isinstance(annotation, type) and annotation != str:
                # The parameter is annotated with a type that isn't str: convert
                # the arg to that type.
                args[i] = annotation(arg)
        elif default != inspect.Parameter.empty:
            if default is not None and not isinstance(default, str):
                # The parameter has a default value that isn't None or a str:
                # convert the arg to the default value's type.
                args[i] = type(default)(arg)

    return args

Response: Convert a list of command arguments to types specified by the handler.


Instruction:
Make a summary of the following Python 3 code
def add_cmd_handler(self, handler_obj):
        for field in dir(handler_obj):
            if field.startswith("cmd_"):
                cmd = field[4:]
                fn = getattr(handler_obj, field)
                if cmd in self.cmds:
                    print('Replacing {} with {}'.format(
                        _handler_name(self.cmds[cmd]), _handler_name(fn)),
                        file=sys.stderr)
                self.cmds[cmd] = fn

Response: Registers a new command handler object.


Instruction:
Implement a Python 3 function for
padding x and y so that the results have the same length.

Response:
def _pad_tensors_to_same_length(x, y):
  """Pad x and y so that the results have the same length (second dimension)."""
  with tf.name_scope("pad_to_same_length"):
    x_length = tf.shape(x)[1]
    y_length = tf.shape(y)[1]

    max_length = tf.maximum(x_length, y_length)

    x = tf.pad(x, [[0, 0], [0, max_length - x_length], [0, 0]])
    y = tf.pad(y, [[0, 0], [0, max_length - y_length]])
    return x, y


Instruction:
Implement a Python 3 function for
calculating the padded cross entropy loss.

Response:
def padded_cross_entropy_loss(logits, labels, smoothing, vocab_size):
  """Calculate cross entropy loss while ignoring padding.

  Args:
    logits: Tensor of size [batch_size, length_logits, vocab_size]
    labels: Tensor of size [batch_size, length_labels]
    smoothing: Label smoothing constant, used to determine the on and off values
    vocab_size: int size of the vocabulary
  Returns:
    Returns a float32 tensor with shape
      [batch_size, max(length_logits, length_labels)]
  """
  with tf.name_scope("loss", [logits, labels]):
    logits, labels = _pad_tensors_to_same_length(logits, labels)

    # Calculate smoothing cross entropy
    with tf.name_scope("smoothing_cross_entropy", [logits, labels]):
      confidence = 1.0 - smoothing
      low_confidence = (1.0 - confidence) / tf.to_float(vocab_size - 1)
      soft_targets = tf.one_hot(
          tf.cast(labels, tf.int32),
          depth=vocab_size,
          on_value=confidence,
          off_value=low_confidence)
      xentropy = tf.nn.softmax_cross_entropy_with_logits_v2(
          logits=logits, labels=soft_targets)

      # Calculate the best (lowest) possible value of cross entropy, and
      # subtract from the cross entropy loss.
      normalizing_constant = -(
          confidence * tf.log(confidence) + tf.to_float(vocab_size - 1) *
          low_confidence * tf.log(low_confidence + 1e-20))
      xentropy -= normalizing_constant

    weights = tf.to_float(tf.not_equal(labels, 0))
    return xentropy * weights, weights


Instruction:
Implement a function in Python 3 to
wrap a metric fn that returns scores and weights as an eval metric fn.

Response:
def _convert_to_eval_metric(metric_fn):
  """Wrap a metric fn that returns scores and weights as an eval metric fn.

  The input metric_fn returns values for the current batch. The wrapper
  aggregates the return values collected over all of the batches evaluated.

  Args:
    metric_fn: function that returns scores and weights for the current batch's
      logits and predicted labels.

  Returns:
    function that aggregates the scores and weights from metric_fn.
  """
  def problem_metric_fn(*args):
    """Returns an aggregation of the metric_fn's returned values."""
    (scores, weights) = metric_fn(*args)

    # The tf.metrics.mean function assures correct aggregation.
    return tf.metrics.mean(scores, weights)
  return problem_metric_fn


Instruction:
Write a Python 3 script to
return dictionary of model evaluation metrics.

Response:
def get_eval_metrics(logits, labels, params):
  """Return dictionary of model evaluation metrics."""
  metrics = {
      "accuracy": _convert_to_eval_metric(padded_accuracy)(logits, labels),
      "accuracy_top5": _convert_to_eval_metric(padded_accuracy_top5)(
          logits, labels),
      "accuracy_per_sequence": _convert_to_eval_metric(
          padded_sequence_accuracy)(logits, labels),
      "neg_log_perplexity": _convert_to_eval_metric(padded_neg_log_perplexity)(
          logits, labels, params.vocab_size),
      "approx_bleu_score": _convert_to_eval_metric(bleu_score)(logits, labels),
      "rouge_2_fscore": _convert_to_eval_metric(rouge_2_fscore)(logits, labels),
      "rouge_L_fscore": _convert_to_eval_metric(rouge_l_fscore)(logits, labels),
  }

  # Prefix each of the metric names with "metrics/". This allows the metric
  # graphs to display under the "metrics" category in TensorBoard.
  metrics = {"metrics/%s" % k: v for k, v in six.iteritems(metrics)}
  return metrics


Instruction:
How would you explain what the following Python 3 function does
def padded_accuracy_topk(logits, labels, k):
  with tf.variable_scope("padded_accuracy_topk", values=[logits, labels]):
    logits, labels = _pad_tensors_to_same_length(logits, labels)
    weights = tf.to_float(tf.not_equal(labels, 0))
    effective_k = tf.minimum(k, tf.shape(logits)[-1])
    _, outputs = tf.nn.top_k(logits, k=effective_k)
    outputs = tf.to_int32(outputs)
    padded_labels = tf.to_int32(labels)
    padded_labels = tf.expand_dims(padded_labels, axis=-1)
    padded_labels += tf.zeros_like(outputs)  # Pad to same shape.
    same = tf.to_float(tf.equal(outputs, padded_labels))
    same_topk = tf.reduce_sum(same, axis=-1)
    return same_topk, weights

Response: Percentage of times that top - k predictions matches labels on non - 0s.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def padded_sequence_accuracy(logits, labels):
  with tf.variable_scope("padded_sequence_accuracy", values=[logits, labels]):
    logits, labels = _pad_tensors_to_same_length(logits, labels)
    weights = tf.to_float(tf.not_equal(labels, 0))
    outputs = tf.to_int32(tf.argmax(logits, axis=-1))
    padded_labels = tf.to_int32(labels)
    not_correct = tf.to_float(tf.not_equal(outputs, padded_labels)) * weights
    axis = list(range(1, len(outputs.get_shape())))
    correct_seq = 1.0 - tf.minimum(1.0, tf.reduce_sum(not_correct, axis=axis))
    return correct_seq, tf.constant(1.0)

Response: Percentage of times that predictions matches labels everywhere ( non - 0 ).


Instruction:
Explain what the following Python 3 code does
def padded_neg_log_perplexity(logits, labels, vocab_size):
  num, den = padded_cross_entropy_loss(logits, labels, 0, vocab_size)
  return -num, den

Response: Average log - perplexity excluding padding 0s. No smoothing.


Instruction:
Explain what the following Python 3 code does
def bleu_score(logits, labels):
  predictions = tf.to_int32(tf.argmax(logits, axis=-1))
  # TODO: Look into removing use of py_func
  bleu = tf.py_func(compute_bleu, (labels, predictions), tf.float32)
  return bleu, tf.constant(1.0)

Response: Approximate BLEU score computation between labels and predictions.


Instruction:
Given the following Python 3 function, write the documentation
def _get_ngrams_with_counter(segment, max_order):
  ngram_counts = collections.Counter()
  for order in xrange(1, max_order + 1):
    for i in xrange(0, len(segment) - order + 1):
      ngram = tuple(segment[i:i + order])
      ngram_counts[ngram] += 1
  return ngram_counts

Response: Extracts all n - grams from a given text segment and returns a Counter containing the number of times each n - gram occurred.


Instruction:
Implement a function in Python 3 to
compute BLEU score of translated segments against one or more references.

Response:
def compute_bleu(reference_corpus, translation_corpus, max_order=4,
                 use_bp=True):
  """Computes BLEU score of translated segments against one or more references.

  Args:
    reference_corpus: list of references for each translation. Each
        reference should be tokenized into a list of tokens.
    translation_corpus: list of translations to score. Each translation
        should be tokenized into a list of tokens.
    max_order: Maximum n-gram order to use when computing BLEU score.
    use_bp: boolean, whether to apply brevity penalty.

  Returns:
    BLEU score.
  """
  reference_length = 0
  translation_length = 0
  bp = 1.0
  geo_mean = 0

  matches_by_order = [0] * max_order
  possible_matches_by_order = [0] * max_order
  precisions = []

  for (references, translations) in zip(reference_corpus, translation_corpus):
    reference_length += len(references)
    translation_length += len(translations)
    ref_ngram_counts = _get_ngrams_with_counter(references, max_order)
    translation_ngram_counts = _get_ngrams_with_counter(translations, max_order)

    overlap = dict((ngram,
                    min(count, translation_ngram_counts[ngram]))
                   for ngram, count in ref_ngram_counts.items())

    for ngram in overlap:
      matches_by_order[len(ngram) - 1] += overlap[ngram]
    for ngram in translation_ngram_counts:
      possible_matches_by_order[len(ngram) - 1] += translation_ngram_counts[
          ngram]

  precisions = [0] * max_order
  smooth = 1.0

  for i in xrange(0, max_order):
    if possible_matches_by_order[i] > 0:
      precisions[i] = float(matches_by_order[i]) / possible_matches_by_order[i]
      if matches_by_order[i] > 0:
        precisions[i] = float(matches_by_order[i]) / possible_matches_by_order[
            i]
      else:
        smooth *= 2
        precisions[i] = 1.0 / (smooth * possible_matches_by_order[i])
    else:
      precisions[i] = 0.0

  if max(precisions) > 0:
    p_log_sum = sum(math.log(p) for p in precisions if p)
    geo_mean = math.exp(p_log_sum / max_order)

  if use_bp:
    ratio = translation_length / reference_length
    bp = math.exp(1 - 1. / ratio) if ratio < 1.0 else 1.0
  bleu = geo_mean * bp
  return np.float32(bleu)


Instruction:
Write a Python 3 function that can
rouge - 2 F1 score computation between labels and predictions.

Response:
def rouge_2_fscore(logits, labels):
  """ROUGE-2 F1 score computation between labels and predictions.

  This is an approximate ROUGE scoring method since we do not glue word pieces
  or decode the ids and tokenize the output.

  Args:
    logits: tensor, model predictions
    labels: tensor, gold output.

  Returns:
    rouge2_fscore: approx rouge-2 f1 score.
  """
  predictions = tf.to_int32(tf.argmax(logits, axis=-1))
  # TODO: Look into removing use of py_func
  rouge_2_f_score = tf.py_func(rouge_n, (predictions, labels), tf.float32)
  return rouge_2_f_score, tf.constant(1.0)


Instruction:
Write a Python 3 script to
compute ROUGE - N f1 score of two text collections of sentences.

Response:
def rouge_n(eval_sentences, ref_sentences, n=2):
  """Computes ROUGE-N f1 score of two text collections of sentences.

  Source: https://www.microsoft.com/en-us/research/publication/
  rouge-a-package-for-automatic-evaluation-of-summaries/

  Args:
    eval_sentences: Predicted sentences.
    ref_sentences: Sentences from the reference set
    n: Size of ngram.  Defaults to 2.

  Returns:
    f1 score for ROUGE-N
  """
  f1_scores = []
  for eval_sentence, ref_sentence in zip(eval_sentences, ref_sentences):
    eval_ngrams = _get_ngrams(n, eval_sentence)
    ref_ngrams = _get_ngrams(n, ref_sentence)
    ref_count = len(ref_ngrams)
    eval_count = len(eval_ngrams)

    # Count the overlapping ngrams between evaluated and reference
    overlapping_ngrams = eval_ngrams.intersection(ref_ngrams)
    overlapping_count = len(overlapping_ngrams)

    # Handle edge case. This isn't mathematically correct, but it's good enough
    if eval_count == 0:
      precision = 0.0
    else:
      precision = float(overlapping_count) / eval_count
    if ref_count == 0:
      recall = 0.0
    else:
      recall = float(overlapping_count) / ref_count
    f1_scores.append(2.0 * ((precision * recall) / (precision + recall + 1e-8)))

  # return overlapping_count / reference_count
  return np.mean(f1_scores, dtype=np.float32)


Instruction:
Can you tell what is the following Python 3 function doing
def rouge_l_fscore(predictions, labels):
  outputs = tf.to_int32(tf.argmax(predictions, axis=-1))
  rouge_l_f_score = tf.py_func(rouge_l_sentence_level, (outputs, labels),
                               tf.float32)
  return rouge_l_f_score, tf.constant(1.0)

Response: ROUGE scores computation between labels and predictions.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def parse_args():
    parser = ArgumentParser(description="PyTorch distributed training launch "
                                        "helper utilty that will spawn up "
                                        "multiple distributed processes")

    # Optional arguments for the launch helper
    parser.add_argument("--nnodes", type=int, default=1,
                        help="The number of nodes to use for distributed "
                             "training")
    parser.add_argument("--node_rank", type=int, default=0,
                        help="The rank of the node for multi-node distributed "
                             "training")
    parser.add_argument("--nproc_per_node", type=int, default=1,
                        help="The number of processes to launch on each node, "
                             "for GPU training, this is recommended to be set "
                             "to the number of GPUs in your system so that "
                             "each process can be bound to a single GPU.")
    parser.add_argument("--master_addr", default="127.0.0.1", type=str,
                        help="Master node (rank 0)'s address, should be either "
                             "the IP address or the hostname of node 0, for "
                             "single node multi-proc training, the "
                             "--master_addr can simply be 127.0.0.1")
    parser.add_argument("--master_port", default=29500, type=int,
                        help="Master node (rank 0)'s free port that needs to "
                             "be used for communciation during distributed "
                             "training")
    parser.add_argument('--no_hyperthreads', action='store_true',
                        help='Flag to disable binding to hyperthreads')
    parser.add_argument('--no_membind', action='store_true',
                        help='Flag to disable memory binding')

    # non-optional arguments for binding
    parser.add_argument("--nsockets_per_node", type=int, required=True,
                        help="Number of CPU sockets on a node")
    parser.add_argument("--ncores_per_socket", type=int, required=True,
                        help="Number of CPU cores per socket")

    # positional
    parser.add_argument("training_script", type=str,
                        help="The full path to the single GPU training "
                             "program/script to be launched in parallel, "
                             "followed by all the arguments for the "
                             "training script")

    # rest from the training program
    parser.add_argument('training_script_args', nargs=REMAINDER)
    return parser.parse_args()

Response: Parse command line options and return a parser object


Instruction:
Implement a function in Python 3 to
download and extract files from compressed archive file.

Response:
def download_and_extract(path, url, input_filename, target_filename):
  """Extract files from downloaded compressed archive file.

  Args:
    path: string directory where the files will be downloaded
    url: url containing the compressed input and target files
    input_filename: name of file containing data in source language
    target_filename: name of file containing data in target language

  Returns:
    Full paths to extracted input and target files.

  Raises:
    OSError: if the the download/extraction fails.
  """
  logging.info('Downloading and extracting data to: %s' % path)
  # Check if extracted files already exist in path
  input_file = find_file(path, input_filename)
  target_file = find_file(path, target_filename)
  if input_file and target_file:
    logging.info("Already downloaded and extracted %s." % url)
    return input_file, target_file

  # Download archive file if it doesn't already exist.
  compressed_file = download_from_url(path, url)

  # Extract compressed files
  logging.info("Extracting %s." % compressed_file)
  with tarfile.open(compressed_file, "r:gz") as corpus_tar:
    corpus_tar.extractall(path)

  # Return filepaths of the requested files.
  input_file = find_file(path, input_filename)
  target_file = find_file(path, target_filename)

  if input_file and target_file:
    return input_file, target_file

  raise OSError("Download/extraction failed for url %s to path %s" %
                (url, path))


Instruction:
Can you generate a brief explanation for the following Python 3 code
def main(unused_argv):
  make_dir(FLAGS.raw_dir)
  make_dir(FLAGS.data_dir)

  # Get paths of download/extracted training and evaluation files.
  print("Step 1/4: Downloading data from source")
  train_files = get_raw_files(FLAGS.raw_dir, _TRAIN_DATA_SOURCES)
  eval_files = get_raw_files(FLAGS.raw_dir, _EVAL_DATA_SOURCES)

Response: Download training and evaluation data for the Transformer model.


Instruction:
Here you have a function in Python 3, explain what it does
def parse_comment_node(comment):
    # Example of a comment node. The resign threshold line appears only
    # for the first move in the game; it gets preprocessed by extract_game_data
    """
    Resign Threshold: -0.88
    -0.0662
    D4 (100) ==> D16 (14) ==> Q16 (3) ==> Q4 (1) ==> Q: -0.07149
    move: action Q U P P-Dir N soft-N p-delta p-rel
    D4 : -0.028, -0.048, 0.020, 0.048, 0.064, 100 0.1096 0.06127 1.27
    D16 : -0.024, -0.043, 0.019, 0.044, 0.059, 96 0.1053 0.06135 1.40
    """

    lines = comment.split('\n')
    if lines[0].startswith('Resign'):
        lines = lines[1:]

    post_Q = float(lines[0])
    debug_rows = []
    comment_splitter = re.compile(r'[ :,]')
    for line in lines[3:]:
        if not line:
            continue
        columns = comment_splitter.split(line)
        columns = list(filter(bool, columns))
        coord, *other_columns = columns
        coord = coords.to_flat(coords.from_gtp(coord))
        debug_rows.append(DebugRow(coord, *map(float, other_columns)))
        if FLAGS.only_top_move:
            break
    return post_Q, debug_rows

Response: Parse a comment node into a post - Q and debug rows.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def forward(self, inputs, lengths):
        x = self.embedder(inputs)

        # bidirectional layer
        x = self.dropout(x)
        x = pack_padded_sequence(x, lengths.cpu().numpy(),
                                 batch_first=self.batch_first)
        x, _ = self.rnn_layers[0](x)
        x, _ = pad_packed_sequence(x, batch_first=self.batch_first)

        # 1st unidirectional layer
        x = self.dropout(x)
        x, _ = self.rnn_layers[1](x)

        # the rest of unidirectional layers,
        # with residual connections starting from 3rd layer
        for i in range(2, len(self.rnn_layers)):
            residual = x
            x = self.dropout(x)
            x, _ = self.rnn_layers[i](x)
            x = x + residual

        return x

Response:
Execute the encoder.

        :param inputs: tensor with indices from the vocabulary
        :param lengths: vector with sequence lengths (excluding padding)

        returns: tensor with encoded sequences


Instruction:
Implement a Python 3 function for
running the given subprocess command in a coroutine.

Response:
async def checked_run(*cmd):
  """Run the given subprocess command in a coroutine.

  Args:
    *cmd: the command to run and its arguments.

  Returns:
    The output that the command wrote to stdout.

  Raises:
    RuntimeError: if the command returns a non-zero result.
  """

  # Start the subprocess.
  logging.info('Running: %s', expand_cmd_str(cmd))
  with logged_timer('{} finished'.format(get_cmd_name(cmd))):
    p = await asyncio.create_subprocess_exec(
        *cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.STDOUT)

    # Stream output from the process stdout.
    chunks = []
    while True:
      chunk = await p.stdout.read(16 * 1024)
      if not chunk:
        break
      chunks.append(chunk)

    # Wait for the process to finish, check it was successful & build stdout.
    await p.wait()
    stdout = b''.join(chunks).decode()[:-1]
    if p.returncode:
      raise RuntimeError('Return code {} from process: {}\n{}'.format(
          p.returncode, expand_cmd_str(cmd), stdout))

    return stdout


Instruction:
How would you explain what the following Python 3 function does
def wait(aws):

  aws_list = aws if isinstance(aws, list) else [aws]
  results = asyncio.get_event_loop().run_until_complete(asyncio.gather(
      *aws_list, return_exceptions=True))
  # If any of the cmds failed, re-raise the error.
  for result in results:
    if isinstance(result, Exception):
      raise result
  return results if isinstance(aws, list) else results[0]

Response: Waits for all of the awaitable objects in aws to finish.


Instruction:
Explain what the following Python 3 code does
def get_subdirs(parent_path):
  entries = os.listdir(parent_path)
  subdirs = [(entry, os.path.join(parent_path, entry))
             for entry in entries
             if os.path.isdir(entry)]
  return subdirs

Response:
Return a list of tuples of direct subdirectories of

    parent_path where each tuple corresponds to one subdirectory.


Instruction:
How would you code a function in Python 3 to
forward the feature maps for each level and return the result.

Response:
def forward(self, x, boxes):
        """
        Arguments:
            x (list[Tensor]): feature maps for each level
            boxes (list[BoxList]): boxes to be used to perform the pooling operation.
        Returns:
            result (Tensor)
        """
        num_levels = len(self.poolers)
        rois = self.convert_to_roi_format(boxes)
        if num_levels == 1:
            return self.poolers[0](x[0], rois)

        levels = self.map_levels(boxes)

        num_rois = len(rois)
        num_channels = x[0].shape[1]
        output_size = self.output_size[0]

        dtype, device = x[0].dtype, x[0].device
        result = torch.zeros(
            (num_rois, num_channels, output_size, output_size),
            dtype=dtype,
            device=device,
        )
        for level, (per_level_feature, pooler) in enumerate(zip(x, self.poolers)):
            idx_in_level = torch.nonzero(levels == level).squeeze(1)
            rois_per_level = rois[idx_in_level]
            result[idx_in_level] = pooler(per_level_feature, rois_per_level)

        return result


Instruction:
Can you implement a function in Python 3 that
augments the audio with sox.

Response:
def augment_audio_with_sox(path, sample_rate, tempo, gain):
    """
    Changes tempo and gain of the recording with sox and loads it.
    """
    with NamedTemporaryFile(suffix=".wav") as augmented_file:
        augmented_filename = augmented_file.name
        sox_augment_params = ["tempo", "{:.3f}".format(tempo), "gain", "{:.3f}".format(gain)]
        sox_params = "sox \"{}\" -r {} -c 1 -b 16 {} {} >/dev/null 2>&1".format(path, sample_rate,
                                                                            augmented_filename,
                                                                            " ".join(sox_augment_params))
        os.system(sox_params)
        y = load_audio(augmented_filename)
        return y


Instruction:
Can you generate the documentation for the following Python 3 function
def load_randomly_augmented_audio(path, sample_rate=16000, tempo_range=(0.85, 1.15),
                                  gain_range=(-6, 8)):
    """
    Picks tempo and gain uniformly, applies it to the utterance by using sox utility.
    Returns the augmented utterance.
    """
    low_tempo, high_tempo = tempo_range
    tempo_value = np.random.uniform(low=low_tempo, high=high_tempo)
    low_gain, high_gain = gain_range
    gain_value = np.random.uniform(low=low_gain, high=high_gain)
    audio = augment_audio_with_sox(path=path, sample_rate=sample_rate,
                                   tempo=tempo_value, gain=gain_value)
    return audio

Response: Load audio from a randomly augmented utterance.


Instruction:
Can you implement a function in Python 3 that
defines how to train evaluate and predict from the transformer model.

Response:
def model_fn(features, labels, mode, params):
  """Defines how to train, evaluate and predict from the transformer model."""
  with tf.variable_scope("model"):
    inputs, targets = features, labels

    # Create model and get output logits.
    model = transformer.Transformer(params, mode == tf.estimator.ModeKeys.TRAIN)

    output = model(inputs, targets)

    # When in prediction mode, the labels/targets is None. The model output
    # is the prediction
    if mode == tf.estimator.ModeKeys.PREDICT:
      return tf.estimator.EstimatorSpec(
          tf.estimator.ModeKeys.PREDICT,
          predictions=output)

    logits = output

    # Calculate model loss.
    xentropy, weights = metrics.padded_cross_entropy_loss(
        logits, targets, params.label_smoothing, params.vocab_size)
    loss = tf.reduce_sum(xentropy * weights) / tf.reduce_sum(weights)

    if mode == tf.estimator.ModeKeys.EVAL:
      return tf.estimator.EstimatorSpec(
          mode=mode, loss=loss, predictions={"predictions": logits},
          eval_metric_ops=metrics.get_eval_metrics(logits, labels, params))
    else:
      train_op = get_train_op(loss, params)
      return tf.estimator.EstimatorSpec(mode=mode, loss=loss, train_op=train_op)


Instruction:
Create a Python 3 function to
calculate learning rate with linear warmup and rsqrt decay.

Response:
def get_learning_rate(learning_rate, hidden_size, learning_rate_warmup_steps):
  """Calculate learning rate with linear warmup and rsqrt decay."""
  with tf.name_scope("learning_rate"):
    warmup_steps = tf.to_float(learning_rate_warmup_steps)
    step = tf.to_float(tf.train.get_or_create_global_step())

    learning_rate *= (hidden_size ** -0.5)
    # Apply linear warmup
    learning_rate *= tf.minimum(1.0, step / warmup_steps)
    # Apply rsqrt decay
    learning_rate *= tf.rsqrt(tf.maximum(step, warmup_steps))

    # Save learning rate value to TensorBoard summary.
    tf.summary.scalar("learning_rate", learning_rate)

    return learning_rate


Instruction:
How would you implement a function in Python 3 that
generates training operation that updates variables based on loss.

Response:
def get_train_op(loss, params):
  """Generate training operation that updates variables based on loss."""
  with tf.variable_scope("get_train_op"):
    mlperf_log.transformer_print(
        key=mlperf_log.OPT_LR_WARMUP_STEPS,
        value=params.learning_rate_warmup_steps)
    learning_rate = get_learning_rate(
        params.learning_rate, params.hidden_size,
        params.learning_rate_warmup_steps)
    log_id = mlperf_log.resnet_print(key=mlperf_log.OPT_LR, deferred=True)
    learning_rate = tf_mlperf_log.log_deferred(op=learning_rate, log_id=log_id,
                                               every_n=100)

    # Create optimizer. Use LazyAdamOptimizer from TF contrib, which is faster
    # than the TF core Adam optimizer.
    mlperf_log.transformer_print(key=mlperf_log.OPT_NAME,
                                 value=mlperf_log.LAZY_ADAM)
    mlperf_log.transformer_print(key=mlperf_log.OPT_HP_ADAM_BETA1,
                                 value=params.optimizer_adam_beta1)
    mlperf_log.transformer_print(key=mlperf_log.OPT_HP_ADAM_BETA2,
                                 value=params.optimizer_adam_beta2)
    mlperf_log.transformer_print(key=mlperf_log.OPT_HP_ADAM_EPSILON,
                                 value=params.optimizer_adam_epsilon)
    optimizer = tf.contrib.opt.LazyAdamOptimizer(
        learning_rate,
        beta1=params.optimizer_adam_beta1,
        beta2=params.optimizer_adam_beta2,
        epsilon=params.optimizer_adam_epsilon)

    # Calculate and apply gradients using LazyAdamOptimizer.
    global_step = tf.train.get_global_step()
    tvars = tf.trainable_variables()
    gradients = optimizer.compute_gradients(
        loss, tvars, colocate_gradients_with_ops=True)
    train_op = optimizer.apply_gradients(
        gradients, global_step=global_step, name="train")

    # Save gradient norm to Tensorboard
    tf.summary.scalar("global_norm/gradient_norm",
                      tf.global_norm(list(zip(*gradients))[0]))

    return train_op


Instruction:
Here you have a function in Python 3, explain what it does
def translate_and_compute_bleu(estimator, subtokenizer, bleu_source, bleu_ref):
  # Create temporary file to store translation.
  tmp = tempfile.NamedTemporaryFile(delete=False)
  tmp_filename = tmp.name

  translate.translate_file(
      estimator, subtokenizer, bleu_source, output_file=tmp_filename,
      print_all_translations=False)

  # Compute uncased and cased bleu scores.
  uncased_score = compute_bleu.bleu_wrapper(bleu_ref, tmp_filename, False)
  cased_score = compute_bleu.bleu_wrapper(bleu_ref, tmp_filename, True)
  os.remove(tmp_filename)
  return uncased_score, cased_score

Response: Translate file and report the cased and uncased bleu scores.


Instruction:
How would you explain what the following Python 3 function does
def evaluate_and_log_bleu(estimator, bleu_writer, bleu_source, bleu_ref):
  subtokenizer = tokenizer.Subtokenizer(
      os.path.join(FLAGS.data_dir, FLAGS.vocab_file))

  uncased_score, cased_score = translate_and_compute_bleu(
      estimator, subtokenizer, bleu_source, bleu_ref)

  print("Bleu score (uncased):", uncased_score)
  print("Bleu score (cased):", cased_score)

  summary = tf.Summary(value=[
      tf.Summary.Value(tag="bleu/uncased", simple_value=uncased_score),
      tf.Summary.Value(tag="bleu/cased", simple_value=cased_score),
  ])

  bleu_writer.add_summary(summary, get_global_step(estimator))
  bleu_writer.flush()
  return uncased_score, cased_score

Response: Calculate and record the BLEU score.


Instruction:
Can you create a Python 3 function that
trains and evaluate model and optionally compute BLEU score.

Response:
def train_schedule(
    estimator, train_eval_iterations, single_iteration_train_steps=None,
    single_iteration_train_epochs=None, bleu_source=None, bleu_ref=None,
    bleu_threshold=None):
  """Train and evaluate model, and optionally compute model's BLEU score.

  **Step vs. Epoch vs. Iteration**

  Steps and epochs are canonical terms used in TensorFlow and general machine
  learning. They are used to describe running a single process (train/eval):
    - Step refers to running the process through a single or batch of examples.
    - Epoch refers to running the process through an entire dataset.

  E.g. training a dataset with 100 examples. The dataset is
  divided into 20 batches with 5 examples per batch. A single training step
  trains the model on one batch. After 20 training steps, the model will have
  trained on every batch in the dataset, or, in other words, one epoch.

  Meanwhile, iteration is used in this implementation to describe running
  multiple processes (training and eval).
    - A single iteration:
      1. trains the model for a specific number of steps or epochs.
      2. evaluates the model.
      3. (if source and ref files are provided) compute BLEU score.

  This function runs through multiple train+eval+bleu iterations.

  Args:
    estimator: tf.Estimator containing model to train.
    train_eval_iterations: Number of times to repeat the train+eval iteration.
    single_iteration_train_steps: Number of steps to train in one iteration.
    single_iteration_train_epochs: Number of epochs to train in one iteration.
    bleu_source: File containing text to be translated for BLEU calculation.
    bleu_ref: File containing reference translations for BLEU calculation.
    bleu_threshold: minimum BLEU score before training is stopped.

  Raises:
    ValueError: if both or none of single_iteration_train_steps and
      single_iteration_train_epochs were defined.
  """
  # Ensure that exactly one of single_iteration_train_steps and
  # single_iteration_train_epochs is defined.
  if single_iteration_train_steps is None:
    if single_iteration_train_epochs is None:
      raise ValueError(
          "Exactly one of single_iteration_train_steps or "
          "single_iteration_train_epochs must be defined. Both were none.")
  else:
    if single_iteration_train_epochs is not None:
      raise ValueError(
          "Exactly one of single_iteration_train_steps or "
          "single_iteration_train_epochs must be defined. Both were defined.")

  evaluate_bleu = bleu_source is not None and bleu_ref is not None

  # Print out training schedule
  print("Training schedule:")
  if single_iteration_train_epochs is not None:
    print("\t1. Train for %d epochs." % single_iteration_train_epochs)
  else:
    print("\t1. Train for %d steps." % single_iteration_train_steps)
  print("\t2. Evaluate model.")
  if evaluate_bleu:
    print("\t3. Compute BLEU score.")
    if bleu_threshold is not None:
      print("Repeat above steps until the BLEU score reaches", bleu_threshold)
  if not evaluate_bleu or bleu_threshold is None:
    print("Repeat above steps %d times." % train_eval_iterations)

  if evaluate_bleu:
    # Set summary writer to log bleu score.
    bleu_writer = tf.summary.FileWriter(
        os.path.join(estimator.model_dir, BLEU_DIR))
    if bleu_threshold is not None:
      # Change loop stopping condition if bleu_threshold is defined.
      train_eval_iterations = INF

  # Loop training/evaluation/bleu cycles
  mlperf_log.transformer_print(key=mlperf_log.TRAIN_LOOP)
  for i in xrange(train_eval_iterations):
    print("Starting iteration", i + 1)

    mlperf_log.transformer_print(key=mlperf_log.TRAIN_EPOCH,
                                 value=i * single_iteration_train_epochs + 1)

    # Train the model for single_iteration_train_steps or until the input fn
    # runs out of examples (if single_iteration_train_steps is None).
    estimator.train(dataset.train_input_fn, steps=single_iteration_train_steps)

    mlperf_log.transformer_print(key=mlperf_log.EVAL_START)
    eval_results = estimator.evaluate(dataset.eval_input_fn)
    print("Evaluation results (iter %d/%d):" % (i + 1, train_eval_iterations),
          eval_results)

    if evaluate_bleu:
      uncased_score, _ = evaluate_and_log_bleu(
          estimator, bleu_writer, bleu_source, bleu_ref)
      if bleu_threshold is not None and uncased_score > bleu_threshold:
        bleu_writer.close()
        break
      mlperf_log.transformer_print(key=mlperf_log.EVAL_TARGET, value=bleu_threshold)
      mlperf_log.transformer_print(key=mlperf_log.EVAL_ACCURACY, value=uncased_score)
    mlperf_log.transformer_print(key=mlperf_log.EVAL_STOP)


Instruction:
Can you tell what is the following Python 3 function doing
def add_membership(self, member, role=github.GithubObject.NotSet):
        assert isinstance(member, github.NamedUser.NamedUser), member
        assert role is github.GithubObject.NotSet or isinstance(
            role, (str, unicode)), role
        if role is not github.GithubObject.NotSet:
            assert role in ['member', 'maintainer']
            put_parameters = {
                "role": role,
            }
        else:
            put_parameters = {
                "role": "member",
            }
        headers, data = self._requester.requestJsonAndCheck(
            "PUT",
            self.url + "/memberships/" + member._identity,
            input=put_parameters
        )

Response: Adds a member to the hierarchy of the user.


Instruction:
Create a Python 3 function to
add the current user s identity to the specified repository.

Response:
def add_to_repos(self, repo):
        """
        :calls: `PUT /teams/:id/repos/:org/:repo <http://developer.github.com/v3/orgs/teams>`_
        :param repo: :class:`github.Repository.Repository`
        :rtype: None
        """
        assert isinstance(repo, github.Repository.Repository), repo
        headers, data = self._requester.requestJsonAndCheck(
            "PUT",
            self.url + "/repos/" + repo._identity
        )


Instruction:
Can you tell what is the following Python 3 function doing
def set_repo_permission(self, repo, permission):
        assert isinstance(repo, github.Repository.Repository), repo
        put_parameters = {
            "permission": permission,
        }
        headers, data = self._requester.requestJsonAndCheck(
            "PUT",
            self.url + "/repos/" + repo._identity,
            input=put_parameters
        )

Response: sets the permission for the specified repository


Instruction:
Make a summary of the following Python 3 code
def edit(self, name, description=github.GithubObject.NotSet, permission=github.GithubObject.NotSet, privacy=github.GithubObject.NotSet):
        assert isinstance(name, (str, unicode)), name
        assert description is github.GithubObject.NotSet or isinstance(description, (str, unicode)), description
        assert permission is github.GithubObject.NotSet or isinstance(permission, (str, unicode)), permission
        assert privacy is github.GithubObject.NotSet or isinstance(privacy, (str, unicode)), privacy
        post_parameters = {
            "name": name,
        }
        if description is not github.GithubObject.NotSet:
            post_parameters["description"] = description
        if permission is not github.GithubObject.NotSet:
            post_parameters["permission"] = permission
        if privacy is not github.GithubObject.NotSet:
            post_parameters["privacy"] = privacy
        headers, data = self._requester.requestJsonAndCheck(
            "PATCH",
            self.url,
            input=post_parameters
        )
        self._useAttributes(data)

Response: Edits the properties of the current object.


Instruction:
Write a Python 3 function that can
return a paginated list of members of the specified role.

Response:
def get_members(self, role=github.GithubObject.NotSet):
        """
        :calls: `GET /teams/:id/members <https://developer.github.com/v3/teams/members/#list-team-members>`_
        :param role: string
        :rtype: :class:`github.PaginatedList.PaginatedList` of :class:`github.NamedUser.NamedUser`
        """
        assert role is github.GithubObject.NotSet or isinstance(role, (str, unicode)), role
        url_parameters = dict()
        if role is not github.GithubObject.NotSet:
            assert role in ['member', 'maintainer', 'all']
            url_parameters["role"] = role
        return github.PaginatedList.PaginatedList(
            github.NamedUser.NamedUser,
            self._requester,
            self.url + "/members",
            url_parameters
        )


Instruction:
Can you generate the documentation for the following Python 3 function
def get_repos(self):
        return github.PaginatedList.PaginatedList(
            github.Repository.Repository,
            self._requester,
            self.url + "/repos",
            None
        )

Response: Returns a paginated list of repos for the current user.


Instruction:
Write a Python 3 function for
returning true if the user has a team in the hierarchy.

Response:
def has_in_members(self, member):
        """
        :calls: `GET /teams/:id/members/:user <http://developer.github.com/v3/orgs/teams>`_
        :param member: :class:`github.NamedUser.NamedUser`
        :rtype: bool
        """
        assert isinstance(member, github.NamedUser.NamedUser), member
        status, headers, data = self._requester.requestJson(
            "GET",
            self.url + "/members/" + member._identity
        )
        return status == 204