Instruction:
Can you create a Python 3 function that
plays out a self - play match.

Response:
def play(network):
    """Plays out a self-play match, returning a MCTSPlayer object containing:
        - the final position
        - the n x 362 tensor of floats representing the mcts search probabilities
        - the n-ary tensor of floats representing the original value-net estimate
          where n is the number of moves in the game
    """
    readouts = FLAGS.num_readouts  # defined in strategies.py
    # Disable resign in 5% of games
    if random.random() < FLAGS.resign_disable_pct:
        resign_threshold = -1.0
    else:
        resign_threshold = None

    player = MCTSPlayer(network, resign_threshold=resign_threshold)

    player.initialize_game()

    # Must run this once at the start to expand the root node.
    first_node = player.root.select_leaf()
    prob, val = network.run(first_node.position)
    first_node.incorporate_results(prob, val, first_node)

    while True:
        start = time.time()
        player.root.inject_noise()
        current_readouts = player.root.N
        # we want to do "X additional readouts", rather than "up to X readouts".
        while player.root.N < current_readouts + readouts:
            player.tree_search()

        if FLAGS.verbose >= 3:
            print(player.root.position)
            print(player.root.describe())

        if player.should_resign():
            player.set_result(-1 * player.root.position.to_play,
                              was_resign=True)
            break
        move = player.pick_move()
        player.play_move(move)
        if player.root.is_done():
            player.set_result(player.root.position.result(), was_resign=False)
            break

        if (FLAGS.verbose >= 2) or (FLAGS.verbose >= 1 and player.root.position.n % 10 == 9):
            print("Q: {:.5f}".format(player.root.Q))
            dur = time.time() - start
            print("%d: %d readouts, %.3f s/100. (%.2f sec)" % (
                player.root.position.n, readouts, dur / readouts * 100.0, dur), flush=True)
        if FLAGS.verbose >= 3:
            print("Played >>",
                  coords.to_gtp(coords.from_flat(player.root.fmove)))

    if FLAGS.verbose >= 2:
        utils.dbg("%s: %.3f" % (player.result_string, player.root.Q))
        utils.dbg(player.root.position, player.root.position.score())

    return player


Instruction:
Given the following Python 3 function, write the documentation
def run_game(load_file, selfplay_dir=None, holdout_dir=None,
             sgf_dir=None, holdout_pct=0.05):
    """Takes a played game and record results and game data."""
    if sgf_dir is not None:
        minimal_sgf_dir = os.path.join(sgf_dir, 'clean')
        full_sgf_dir = os.path.join(sgf_dir, 'full')
        utils.ensure_dir_exists(minimal_sgf_dir)
        utils.ensure_dir_exists(full_sgf_dir)
    if selfplay_dir is not None:
        utils.ensure_dir_exists(selfplay_dir)
        utils.ensure_dir_exists(holdout_dir)

    with utils.logged_timer("Loading weights from %s ... " % load_file):
        network = dual_net.DualNetwork(load_file)

    with utils.logged_timer("Playing game"):
        player = play(network)

    output_name = '{}-{}'.format(int(time.time()), socket.gethostname())
    game_data = player.extract_data()
    if sgf_dir is not None:
        with gfile.GFile(os.path.join(minimal_sgf_dir, '{}.sgf'.format(output_name)), 'w') as f:
            f.write(player.to_sgf(use_comments=False))
        with gfile.GFile(os.path.join(full_sgf_dir, '{}.sgf'.format(output_name)), 'w') as f:
            f.write(player.to_sgf())

    tf_examples = preprocessing.make_dataset_from_selfplay(game_data)

    if selfplay_dir is not None:
        # Hold out 5% of games for validation.
        if random.random() < holdout_pct:
            fname = os.path.join(holdout_dir,
                                 "{}.tfrecord.zz".format(output_name))
        else:
            fname = os.path.join(selfplay_dir,
                                 "{}.tfrecord.zz".format(output_name))

        preprocessing.write_tf_examples(fname, tf_examples)

Response: Takes a played game and record results and game data.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def main(argv):
    del argv  # Unused
    flags.mark_flag_as_required('load_file')

    run_game(
        load_file=FLAGS.load_file,
        selfplay_dir=FLAGS.selfplay_dir,
        holdout_dir=FLAGS.holdout_dir,
        holdout_pct=FLAGS.holdout_pct,
        sgf_dir=FLAGS.sgf_dir)

Response: Entry point for running one selfplay game.


Instruction:
Write a Python 3 script for
applying algorith 2 in https://arxiv.org/pdf/1901.08910.pdf. Args: usv: matrix to reduce given in SVD form with the spectrum s in increasing order. num_rows: number of rows in the output matrix. num_cols: number of columns in the output matrix. Returns: A resized version of (u, s, v) whose non zero singular values will be identical to the largest singular values in s.

Response:
def resize_matrix(usv, num_rows, num_cols):
  """Apply algorith 2 in https://arxiv.org/pdf/1901.08910.pdf.

  Args:
    usv: matrix to reduce given in SVD form with the spectrum s in
      increasing order.
    num_rows: number of rows in the output matrix.
    num_cols: number of columns in the output matrix.
  Returns:
    A resized version of (u, s, v) whose non zero singular values will be
      identical to the largest singular values in s.
  """
  u, s, v = usv
  k = min(num_rows, num_cols)

  u_random_proj = transform.resize(u[:, :k], (num_rows, k))
  v_random_proj = transform.resize(v[:k, :], (k, num_cols))

  u_random_proj_orth = _closest_column_orthogonal_matrix(u_random_proj)
  v_random_proj_orth = _closest_column_orthogonal_matrix(v_random_proj.T).T

  return np.matmul(u_random_proj_orth,
                   np.matmul(np.diag(s[::-1][:k]), v_random_proj_orth))


Instruction:
Write a Python 3 script for
folding all values of the matrix into [ 0 1 ).

Response:
def normalize_matrix(matrix):
  """Fold all values of the matrix into [0, 1]."""
  abs_matrix = np.abs(matrix.copy())
  return abs_matrix / abs_matrix.max()


Instruction:
Explain what the following Python 3 code does
def _get_sorted_inputs(filename):
  with tf.gfile.Open(filename) as f:
    records = f.read().split("\n")
    inputs = [record.strip() for record in records]
    if not inputs[-1]:
      inputs.pop()

  input_lens = [(i, len(line.split())) for i, line in enumerate(inputs)]
  sorted_input_lens = sorted(input_lens, key=lambda x: x[1], reverse=True)

  sorted_inputs = []
  sorted_keys = {}
  for i, (index, _) in enumerate(sorted_input_lens):
    sorted_inputs.append(inputs[index])
    sorted_keys[index] = i
  return sorted_inputs, sorted_keys

Response: Read and sort lines from the file sorted by decreasing length.


Instruction:
Given the following Python 3 function, write the documentation
def _trim_and_decode(ids, subtokenizer):
  try:
    index = list(ids).index(tokenizer.EOS_ID)
    return subtokenizer.decode(ids[:index])
  except ValueError:  # No EOS found in sequence
    return subtokenizer.decode(ids)

Response: Trim EOS and PAD tokens from ids and decode to return a string.


Instruction:
Implement a function in Python 3 to
translate lines in file and save to output file if specified.

Response:
def translate_file(
    estimator, subtokenizer, input_file, output_file=None,
    print_all_translations=True):
  """Translate lines in file, and save to output file if specified.

  Args:
    estimator: tf.Estimator used to generate the translations.
    subtokenizer: Subtokenizer object for encoding and decoding source and
       translated lines.
    input_file: file containing lines to translate
    output_file: file that stores the generated translations.
    print_all_translations: If true, all translations are printed to stdout.

  Raises:
    ValueError: if output file is invalid.
  """
  batch_size = _DECODE_BATCH_SIZE

  # Read and sort inputs by length. Keep dictionary (original index-->new index
  # in sorted list) to write translations in the original order.
  sorted_inputs, sorted_keys = _get_sorted_inputs(input_file)
  num_decode_batches = (len(sorted_inputs) - 1) // batch_size + 1

  def input_generator():
    """Yield encoded strings from sorted_inputs."""
    for i, line in enumerate(sorted_inputs):
      if i % batch_size == 0:
        batch_num = (i // batch_size) + 1

        print("Decoding batch %d out of %d." % (batch_num, num_decode_batches))
      yield _encode_and_add_eos(line, subtokenizer)

  def input_fn():
    """Created batched dataset of encoded inputs."""
    ds = tf.data.Dataset.from_generator(
        input_generator, tf.int64, tf.TensorShape([None]))
    ds = ds.padded_batch(batch_size, [None])
    return ds

  translations = []
  for i, prediction in enumerate(estimator.predict(input_fn)):
    translation = _trim_and_decode(prediction["outputs"], subtokenizer)
    translations.append(translation)

    if print_all_translations:
      print("Translating:")
      print("\tInput: %s" % sorted_inputs[i])
      print("\tOutput: %s\n" % translation)
      print("=" * 100)

  # Write translations in the order they appeared in the original file.
  if output_file is not None:
    if tf.gfile.IsDirectory(output_file):
      raise ValueError("File output is a directory, will not save outputs to "
                       "file.")
    tf.logging.info("Writing to file %s" % output_file)
    with tf.gfile.Open(output_file, "w") as f:
      for index in xrange(len(sorted_keys)):
        f.write("%s\n" % translations[sorted_keys[index]])


Instruction:
Here you have a function in Python 3, explain what it does
def translate_text(estimator, subtokenizer, txt):
  encoded_txt = _encode_and_add_eos(txt, subtokenizer)

  def input_fn():
    ds = tf.data.Dataset.from_tensors(encoded_txt)
    ds = ds.batch(_DECODE_BATCH_SIZE)
    return ds

  predictions = estimator.predict(input_fn)
  translation = next(predictions)["outputs"]
  translation = _trim_and_decode(translation, subtokenizer)
  print("Translation of \"%s\": \"%s\"" % (txt, translation))

Response: Translate a single string.


Instruction:
Write a Python 3 script to
pad the vocabulary to a multiple of pad tokens.

Response:
def pad_vocabulary(self, vocab, pad):
        """
        Pads vocabulary to a multiple of 'pad' tokens.

        :param vocab: list with vocabulary
        :param pad: integer
        """
        vocab_size = len(vocab)
        padded_vocab_size = (vocab_size + pad - 1) // pad * pad
        for i in range(0, padded_vocab_size - vocab_size):
            token = f'madeupword{i:04d}'
            vocab.append(token)
        assert len(vocab) % pad == 0


Instruction:
Given the following Python 3 function, write the documentation
def segment(self, line):
        line = line.strip().split()
        entry = [self.tok2idx[i] for i in line]
        entry = [config.BOS] + entry + [config.EOS]
        return entry

Response: Tokenizes a single sentence and adds special BOS and EOS tokens.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def detokenize(self, inputs, delim=' '):
        detok = delim.join([self.idx2tok[idx] for idx in inputs])
        detok = detok.replace(self.separator + ' ', '')
        detok = detok.replace(self.separator, '')

        detok = detok.replace(config.BOS_TOKEN, '')
        detok = detok.replace(config.EOS_TOKEN, '')
        detok = detok.replace(config.PAD_TOKEN, '')
        detok = detok.strip()
        return detok

Response: Detokenizes a single sentence and removes token separator characters.


Instruction:
Can you generate the documentation for the following Python 3 function
def forward(self, x):
        last_inner = getattr(self, self.inner_blocks[-1])(x[-1])
        results = []
        results.append(getattr(self, self.layer_blocks[-1])(last_inner))
        for feature, inner_block, layer_block in zip(
            x[:-1][::-1], self.inner_blocks[:-1][::-1], self.layer_blocks[:-1][::-1]
        ):
            if not inner_block:
                continue
            inner_top_down = F.interpolate(last_inner, scale_factor=2, mode="nearest")
            inner_lateral = getattr(self, inner_block)(feature)
            # TODO use size instead of scale to make it robust to different sizes
            # inner_top_down = F.upsample(last_inner, size=inner_lateral.shape[-2:],
            # mode='bilinear', align_corners=False)
            last_inner = inner_lateral + inner_top_down
            results.insert(0, getattr(self, layer_block)(last_inner))

        if isinstance(self.top_blocks, LastLevelP6P7):
            last_results = self.top_blocks(x[-1], results[-1])
            results.extend(last_results)
        elif isinstance(self.top_blocks, LastLevelMaxPool):
            last_results = self.top_blocks(results[-1])
            results.extend(last_results)

        return tuple(results)

Response: Forward the feature maps to the next layer.


Instruction:
Given the following Python 3 function, write the documentation
def upload_benchmark_run(self, dataset_name, table_name, run_id):
    expected_file = os.path.join(
        self._logging_dir, logger.BENCHMARK_RUN_LOG_FILE_NAME)
    with tf.gfile.GFile(expected_file) as f:
      benchmark_json = json.load(f)
      benchmark_json["model_id"] = run_id
      table_ref = self._bq_client.dataset(dataset_name).table(table_name)
      errors = self._bq_client.insert_rows_json(table_ref, [benchmark_json])
      if errors:
        tf.logging.error(
            "Failed to upload benchmark info to bigquery: {}".format(errors))

Response: Uploads benchmark run information to Bigquery.


Instruction:
Write a Python 3 script to
upload metric information to Bigquery.

Response:
def upload_metric(self, dataset_name, table_name, run_id):
    """Upload metric information to Bigquery.

    Args:
      dataset_name: string, the name of bigquery dataset where the data will be
        uploaded.
      table_name: string, the name of bigquery table under the dataset where
        the metric data will be uploaded. This is different from the
        benchmark_run table.
      run_id: string, a unique ID that will be attached to the data, usually
        this is a UUID4 format. This should be the same as the benchmark run_id.
    """
    expected_file = os.path.join(
        self._logging_dir, logger.METRIC_LOG_FILE_NAME)
    with tf.gfile.GFile(expected_file) as f:
      lines = f.readlines()
      metrics = []
      for line in filter(lambda l: l.strip(), lines):
        metric = json.loads(line)
        metric["run_id"] = run_id
        metrics.append(metric)
      table_ref = self._bq_client.dataset(dataset_name).table(table_name)
      errors = self._bq_client.insert_rows_json(table_ref, metrics)
      if errors:
        tf.logging.error(
            "Failed to upload benchmark info to bigquery: {}".format(errors))


Instruction:
Here you have a function in Python 3, explain what it does
def bleu_wrapper(ref_filename, hyp_filename, case_sensitive=False):
  ref_lines = tf.gfile.Open(ref_filename).read().strip().splitlines()
  hyp_lines = tf.gfile.Open(hyp_filename).read().strip().splitlines()

  if len(ref_lines) != len(hyp_lines):
    raise ValueError("Reference and translation files have different number of "
                     "lines.")
  if not case_sensitive:
    ref_lines = [x.lower() for x in ref_lines]
    hyp_lines = [x.lower() for x in hyp_lines]
  ref_tokens = [bleu_tokenize(x) for x in ref_lines]
  hyp_tokens = [bleu_tokenize(x) for x in hyp_lines]
  return metrics.compute_bleu(ref_tokens, hyp_tokens) * 100

Response: Compute BLEU for two files.


Instruction:
Can you generate the documentation for the following Python 3 function
def play_match(black_model, white_model, games, sgf_dir):
    with utils.logged_timer("Loading weights"):
        black_net = dual_net.DualNetwork(black_model)
        white_net = dual_net.DualNetwork(white_model)

    readouts = FLAGS.num_readouts

    black = MCTSPlayer(black_net, two_player_mode=True)
    white = MCTSPlayer(white_net, two_player_mode=True)

    black_name = os.path.basename(black_net.save_file)
    white_name = os.path.basename(white_net.save_file)

    for i in range(games):
        num_move = 0  # The move number of the current game

        for player in [black, white]:
            player.initialize_game()
            first_node = player.root.select_leaf()
            prob, val = player.network.run(first_node.position)
            first_node.incorporate_results(prob, val, first_node)

        while True:
            start = time.time()
            active = white if num_move % 2 else black
            inactive = black if num_move % 2 else white

            current_readouts = active.root.N
            while active.root.N < current_readouts + readouts:
                active.tree_search()

            # print some stats on the search
            if FLAGS.verbose >= 3:
                print(active.root.position)

            # First, check the roots for hopeless games.
            if active.should_resign():  # Force resign
                active.set_result(-1 *
                                  active.root.position.to_play, was_resign=True)
                inactive.set_result(
                    active.root.position.to_play, was_resign=True)

            if active.is_done():
                fname = "{:d}-{:s}-vs-{:s}-{:d}.sgf".format(int(time.time()),
                                                            white_name, black_name, i)
                active.set_result(active.root.position.result(), was_resign=False)
                with gfile.GFile(os.path.join(sgf_dir, fname), 'w') as _file:
                    sgfstr = sgf_wrapper.make_sgf(active.position.recent,
                                                  active.result_string, black_name=black_name,
                                                  white_name=white_name)
                    _file.write(sgfstr)
                print("Finished game", i, active.result_string)
                break

            move = active.pick_move()
            active.play_move(move)
            inactive.play_move(move)

            dur = time.time() - start
            num_move += 1

            if (FLAGS.verbose > 1) or (FLAGS.verbose == 1 and num_move % 10 == 9):
                timeper = (dur / readouts) * 100.0
                print(active.root.position)
                print("%d: %d readouts, %.3f s/100. (%.2f sec)" % (num_move,
                                                                   readouts,
                                                                   timeper,
                                                                   dur))

Response: Plays matches between two neural nets.


Instruction:
Make a summary of the following Python 3 code
def main(argv):
    _, black_model, white_model = argv
    utils.ensure_dir_exists(FLAGS.eval_sgf_dir)
    play_match(black_model, white_model, FLAGS.num_evaluation_games, FLAGS.eval_sgf_dir)

Response: Play matches between two neural nets.


Instruction:
Explain what the following Python 3 code does
def main():
    args = parse_args()

    c = Counter()
    skipped = 0
    valid = 0
    data1 = []
    data2 = []

    with open(args.file1) as f1, open(args.file2) as f2:
        for idx, lines in enumerate(zip(f1, f2)):
            line1, line2 = lines
            if idx % 100000 == 1:
                print('Processed {} lines'.format(idx))
            try:
                line1.encode('latin1')
                line2.encode('latin1')
            except UnicodeEncodeError:
                skipped += 1
            else:
                data1.append(line1)
                data2.append(line2)
                valid += 1
                c.update(line1)

    ratio = valid / (skipped + valid)
    print('Skipped: {}, Valid: {}, Valid ratio {}'.format(skipped, valid, ratio))
    print('Character frequency:', c)

    save_output(args.file1, data1)
    save_output(args.file2, data2)

Response: This function is used to get the most likely unicode glyphs and pairs of sentences which can t be decoded by latin - 1 encoder.


Instruction:
Write a Python 3 script for
encoding a set of proposals with respect to some reference boxes.

Response:
def encode(self, reference_boxes, proposals):
        """
        Encode a set of proposals with respect to some
        reference boxes

        Arguments:
            reference_boxes (Tensor): reference boxes
            proposals (Tensor): boxes to be encoded
        """

        TO_REMOVE = 1  # TODO remove
        ex_widths = proposals[:, 2] - proposals[:, 0] + TO_REMOVE
        ex_heights = proposals[:, 3] - proposals[:, 1] + TO_REMOVE
        ex_ctr_x = proposals[:, 0] + 0.5 * ex_widths
        ex_ctr_y = proposals[:, 1] + 0.5 * ex_heights

        gt_widths = reference_boxes[:, 2] - reference_boxes[:, 0] + TO_REMOVE
        gt_heights = reference_boxes[:, 3] - reference_boxes[:, 1] + TO_REMOVE
        gt_ctr_x = reference_boxes[:, 0] + 0.5 * gt_widths
        gt_ctr_y = reference_boxes[:, 1] + 0.5 * gt_heights

        wx, wy, ww, wh = self.weights
        targets_dx = wx * (gt_ctr_x - ex_ctr_x) / ex_widths
        targets_dy = wy * (gt_ctr_y - ex_ctr_y) / ex_heights
        targets_dw = ww * torch.log(gt_widths / ex_widths)
        targets_dh = wh * torch.log(gt_heights / ex_heights)

        targets = torch.stack((targets_dx, targets_dy, targets_dw, targets_dh), dim=1)
        return targets


Instruction:
Can you implement a function in Python 3 that
launches an evaluator job.

Response:
def launch_eval_job(tag, m1_path, m2_path, job_name, completions):
    """Launches an evaluator job.
    tag: name for this eval job (used as top level folder name)
    m1_path, m2_path: full gs:// paths to the .pb files to match up
    job_name: string, appended to the container, used to differentiate the job
    names (e.g. 'minigo-cc-evaluator-v5-123-v7-456')
    completions: the number of completions desired (each completion is 2 games)
    """
    print()
    if not re.match(r'[a-z0-9-]*$', tag, re.I):
        print("{} is not a valid tag".format(tag))
        return

    # TODO: Change to minigo-pub
    sgf_bucket_path = "sethtroisi-sandbox/experiments/eval/" + tag
    assert not sgf_bucket_path.startswith("gs://"), bucket_pat
    bucket_path = "gs://" + sgf_bucket_path

    metadata_path = os.path.join(bucket_path, 'metadata')
    assert not gfile.Exists(metadata_path), "Already exists"

    TS=str(int(time.time()))
    metadata = {
        'timestamp': TS,
        'date': datetime.datetime.now().isoformat(' '),
        'model1': os.path.basename(m1_path),
        'model2': os.path.basename(m2_path),
        'model1_path': m1_path,
        'model2_path': m2_path,
        'job_name': job_name,
        'completions': completions,
        'launch_eval_version' : LAUNCH_EVAL_VERSION,
    }

    job_conf, resp_bw, resp_wb = launch_eval.launch_eval_job(
        m1_path, m2_path, job_name, sgf_bucket_path, completions)

    if not (resp_bw and resp_wb):
        print("launch_eval.py failed")
        print(job_conf)
        print(resp_bw)
        print(resp_wb)
        print()
        assert False

    # Jobs were launched, record metadata to GCS.
    with gfile.GFile(metadata_path, "w") as metadata_file:
        json.dump(metadata, metadata_file)

    with gfile.GFile(os.path.join(bucket_path, 'commands'), "w") as f:
        f.write(str(sys.argv) + "\n")

    with gfile.GFile(os.path.join(bucket_path, 'job_conf'), "w") as f:
        f.write(str(job_conf) + "\n")


Instruction:
How would you explain what the following Python 3 function does
def launch_eval_job(m1_path, m2_path, job_name,
        bucket_name, completions=5, flags_path=None):
    """Launches an evaluator job.
    m1_path, m2_path: full gs:// paths to the .pb files to match up
    job_name: string, appended to the container, used to differentiate the job
    names (e.g. 'minigo-cc-evaluator-v5-123-v7-456')
    bucket_name: Where to write the sgfs, passed into the job as $BUCKET_NAME
    completions: the number of completions desired
    flags_path: the path to the eval flagfile to use (if any)
    """
    if not all([m1_path, m2_path, job_name, bucket_name]):
        print("Provide all of m1_path, m2_path, job_name, and bucket_name "
              "params")
        return
    api_instance = get_api()

    raw_job_conf = open("cluster/evaluator/cc-evaluator.yaml").read()

    if flags_path:
        os.environ['EVAL_FLAGS_PATH'] = flags_path
    else:
        os.environ['EVAL_FLAGS_PATH'] = ""
    os.environ['BUCKET_NAME'] = bucket_name
    os.environ['MODEL_BLACK'] = m1_path
    os.environ['MODEL_WHITE'] = m2_path
    os.environ['JOBNAME'] = job_name
    env_job_conf = os.path.expandvars(raw_job_conf)

    job_conf = yaml.load(env_job_conf)
    job_conf['spec']['completions'] = completions

    response = api_instance.create_namespaced_job('default', body=job_conf)
    return job_conf, response

Response: Launches an evaluator job.


Instruction:
Make a summary of the following Python 3 code
def same_run_eval(black_num=0, white_num=0, completions=4):
    if black_num <= 0 or white_num <= 0:
        print("Need real model numbers")
        return

    b = fsdb.get_model(black_num)
    w = fsdb.get_model(white_num)

    b_model_path = os.path.join(fsdb.models_dir(), b)
    w_model_path = os.path.join(fsdb.models_dir(), w)
    flags_path = fsdb.eval_flags_path()

    obj = launch_eval_job(b_model_path + ".pb",
                           w_model_path + ".pb",
                           "{:d}-{:d}".format(black_num, white_num),
                           bucket_name=flags.FLAGS.bucket_name,
                           flags_path=flags_path,
                           completions=completions)

    # Fire spams the retval to stdout, so...
    return "{} job launched ok".format(obj[1].metadata.name)

Response: Spawns a job matching up two models from the same run


Instruction:
Can you generate the documentation for the following Python 3 function
def _append_pairs(new_pairs):
    desired_pairs = restore_pairs() or []
    desired_pairs += new_pairs
    print("Adding {} new pairs, queue has {} pairs".format(len(new_pairs), len(desired_pairs)))
    save_pairs(desired_pairs)

Response: Add new pairs to the pairlist


Instruction:
Can you write a function in Python 3 where it
adds the top pairs to the working pairlist.

Response:
def add_top_pairs(dry_run=False, pair_now=False):
    """ Pairs up the top twenty models against each other.
    #1 plays 2,3,4,5, #2 plays 3,4,5,6 etc. for a total of 15*4 matches.

    Default behavior is to add the pairs to the working pairlist.
    `pair_now` will immediately create the pairings on the cluster.
    `dry_run` makes it only print the pairings that would be added
    """
    top = ratings.top_n(15)
    new_pairs = []
    for idx, t in enumerate(top[:10]):
        new_pairs += [[t[0], o[0]] for o in top[idx+1:idx+5]]

    if dry_run:
        print(new_pairs)
        return

    if pair_now:
        maybe_enqueue(new_pairs)
    else:
        _append_pairs(new_pairs)


Instruction:
Can you tell what is the following Python 3 function doing
def zoo_loop(sgf_dir=None, max_jobs=40):
    desired_pairs = restore_pairs() or []
    random.shuffle(desired_pairs)
    last_model_queued = restore_last_model()

    if sgf_dir:
        sgf_dir = os.path.abspath(sgf_dir)

    api_instance = get_api()
    toggle = True
    try:
        while True:
            last_model = fsdb.get_latest_pb()[0]
            if last_model_queued < last_model:
                print("Adding models {} to {} to be scheduled".format(
                    last_model_queued+1, last_model))
                for m in reversed(range(last_model_queued+1, last_model+1)):
                    desired_pairs += make_pairs_for_model(m)
                last_model_queued = last_model
                save_last_model(last_model)

            cleanup(api_instance)
            random.shuffle(desired_pairs)
            r = api_instance.list_job_for_all_namespaces()
            if r.items:
                tasks = sum([item.spec.completions for item in r.items])
            else:
                tasks = 0
            if tasks < MAX_TASKS:
                if len(desired_pairs) == 0:
                    if sgf_dir:
                        if tasks > MIN_TASKS:
                            time.sleep(60)
                            continue
                        print("Out of pairs!  Syncing new eval games...")
                        ratings.sync(sgf_dir)
                        print("Updating ratings and getting suggestions...")
                        if toggle:
                          print("Pairing the top of the table.")
                          add_top_pairs()
                        else:
                          print("Pairing the least-known models.")
                          add_uncertain_pairs()
                        toggle = not toggle
                        for modelnum, rate in ratings.top_n():
                          print("{:>30}: {:0.3f} ({:0.3f})".format(modelnum, rate[0], rate[1]))
                        desired_pairs = restore_pairs() or []
                    else:
                        print("Out of pairs.  Sleeping ({} remain)".format(len(r.items)))
                        time.sleep(600)
                        continue

                next_pair = desired_pairs.pop()
                failed = maybe_enqueue([next_pair])
                if failed != []:
                    desired_pairs.extend(failed)
                save_pairs(sorted(desired_pairs))
                save_last_model(last_model)
                time.sleep(1)

            else:
                print("{}\t {} finished / {} requested. "
                      "({} jobs, {} pairs to be scheduled)".format(
                      time.strftime("%I:%M:%S %p"),
                      sum([i.status.succeeded or 0 for i in r.items]),
                      tasks, len(r.items), len(desired_pairs)))
                time.sleep(60)
    except:
        print("Unfinished pairs:")
        print(sorted(desired_pairs))
        save_pairs(sorted(desired_pairs))
        save_last_model(last_model)
        raise

Response: This function loops through the list of possible match jobs and creates new ones and deletes them if they are too long.


Instruction:
Can you generate the documentation for the following Python 3 function
def cleanup(api_instance=None):
    api = api_instance or get_api()
    r = api.list_job_for_all_namespaces()
    delete_opts = kubernetes.client.V1DeleteOptions(
            propagation_policy="Background")
    for job in r.items:
        if job.status.succeeded == job.spec.completions:
            print(job.metadata.name, "finished!")
            api.delete_namespaced_job(
                job.metadata.name, 'default', body=delete_opts)

Response: Remove completed jobs from the cluster


Instruction:
How would you code a function in Python 3 to
convert to png and save json with path.

Response:
def convert_coco_stuff_mat(data_dir, out_dir):
    """Convert to png and save json with path. This currently only contains
    the segmentation labels for objects+stuff in cocostuff - if we need to
    combine with other labels from original COCO that will be a TODO."""
    sets = ['train', 'val']
    categories = []
    json_name = 'coco_stuff_%s.json'
    ann_dict = {}
    for data_set in sets:
        file_list = os.path.join(data_dir, '%s.txt')
        images = []
        with open(file_list % data_set) as f:
            for img_id, img_name in enumerate(f):
                img_name = img_name.replace('coco', 'COCO').strip('\n')
                image = {}
                mat_file = os.path.join(
                    data_dir, 'annotations/%s.mat' % img_name)
                data = h5py.File(mat_file, 'r')
                labelMap = data.get('S')
                if len(categories) == 0:
                    labelNames = data.get('names')
                    for idx, n in enumerate(labelNames):
                        categories.append(
                            {"id": idx, "name": ''.join(chr(i) for i in data[
                                n[0]])})
                    ann_dict['categories'] = categories
                scipy.misc.imsave(
                    os.path.join(data_dir, img_name + '.png'), labelMap)
                image['width'] = labelMap.shape[0]
                image['height'] = labelMap.shape[1]
                image['file_name'] = img_name
                image['seg_file_name'] = img_name
                image['id'] = img_id
                images.append(image)
        ann_dict['images'] = images
        print("Num images: %s" % len(images))
        with open(os.path.join(out_dir, json_name % data_set), 'wb') as outfile:
            outfile.write(json.dumps(ann_dict))


Instruction:
How would you code a function in Python 3 to
convert from cityscapes format to COCO instance seg format - polygons

Response:
def convert_cityscapes_instance_only(
        data_dir, out_dir):
    """Convert from cityscapes format to COCO instance seg format - polygons"""
    sets = [
        'gtFine_val',
        'gtFine_train',
        'gtFine_test',

        # 'gtCoarse_train',
        # 'gtCoarse_val',
        # 'gtCoarse_train_extra'
    ]
    ann_dirs = [
        'gtFine_trainvaltest/gtFine/val',
        'gtFine_trainvaltest/gtFine/train',
        'gtFine_trainvaltest/gtFine/test',

        # 'gtCoarse/train',
        # 'gtCoarse/train_extra',
        # 'gtCoarse/val'
    ]
    json_name = 'instancesonly_filtered_%s.json'
    ends_in = '%s_polygons.json'
    img_id = 0
    ann_id = 0
    cat_id = 1
    category_dict = {}

    category_instancesonly = [
        'person',
        'rider',
        'car',
        'truck',
        'bus',
        'train',
        'motorcycle',
        'bicycle',
    ]

    for data_set, ann_dir in zip(sets, ann_dirs):
        print('Starting %s' % data_set)
        ann_dict = {}
        images = []
        annotations = []
        ann_dir = os.path.join(data_dir, ann_dir)
        for root, _, files in os.walk(ann_dir):
            for filename in files:
                if filename.endswith(ends_in % data_set.split('_')[0]):
                    if len(images) % 50 == 0:
                        print("Processed %s images, %s annotations" % (
                            len(images), len(annotations)))
                    json_ann = json.load(open(os.path.join(root, filename)))
                    image = {}
                    image['id'] = img_id
                    img_id += 1

                    image['width'] = json_ann['imgWidth']
                    image['height'] = json_ann['imgHeight']
                    image['file_name'] = filename[:-len(
                        ends_in % data_set.split('_')[0])] + 'leftImg8bit.png'
                    image['seg_file_name'] = filename[:-len(
                        ends_in % data_set.split('_')[0])] + \
                        '%s_instanceIds.png' % data_set.split('_')[0]
                    images.append(image)

                    fullname = os.path.join(root, image['seg_file_name'])
                    objects = cs.instances2dict_with_polygons(
                        [fullname], verbose=False)[fullname]

                    for object_cls in objects:
                        if object_cls not in category_instancesonly:
                            continue  # skip non-instance categories

                        for obj in objects[object_cls]:
                            if obj['contours'] == []:
                                print('Warning: empty contours.')
                                continue  # skip non-instance categories

                            len_p = [len(p) for p in obj['contours']]
                            if min(len_p) <= 4:
                                print('Warning: invalid contours.')
                                continue  # skip non-instance categories

                            ann = {}
                            ann['id'] = ann_id
                            ann_id += 1
                            ann['image_id'] = image['id']
                            ann['segmentation'] = obj['contours']

                            if object_cls not in category_dict:
                                category_dict[object_cls] = cat_id
                                cat_id += 1
                            ann['category_id'] = category_dict[object_cls]
                            ann['iscrowd'] = 0
                            ann['area'] = obj['pixelCount']
                            ann['bbox'] = bboxs_util.xyxy_to_xywh(
                                segms_util.polys_to_boxes(
                                    [ann['segmentation']])).tolist()[0]

                            annotations.append(ann)

        ann_dict['images'] = images
        categories = [{"id": category_dict[name], "name": name} for name in
                      category_dict]
        ann_dict['categories'] = categories
        ann_dict['annotations'] = annotations
        print("Num categories: %s" % len(categories))
        print("Num images: %s" % len(images))
        print("Num annotations: %s" % len(annotations))
        with open(os.path.join(out_dir, json_name % data_set), 'w') as outfile:
            outfile.write(json.dumps(ann_dict))


Instruction:
Implement a function in Python 3 to
get the path to the MG file for a given model run and model number.

Response:
def get_mg_path(model_run, model_num):
    """
    model_run = integer, e.g. 15, 16, corresponding to the v-number
    model_num = integer, e.g 939, for the model number in that run
    """
    fsdb.switch_base("minigo-pub/v{:d}-19x19".format(model_run))
    model = fsdb.get_model(model_num)
    return os.path.join(fsdb.models_dir(), model)


Instruction:
Can you generate the documentation for the following Python 3 function
def make_single_array(ds, batch_size=8*1024):
    if isinstance(ds.output_types, tuple) or isinstance(ds.output_shapes, tuple):
        raise ValueError('Dataset must have a single type and shape')
    nshapes = len(ds.output_shapes)
    if nshapes > 0:
        raise ValueError('Dataset must be comprised of scalars (TensorShape=[])')
    batches = []
    with tf.Session() as sess:
        ds = ds.batch(batch_size)
        iterator = ds.make_initializable_iterator()
        sess.run(iterator.initializer)
        get_next = iterator.get_next()
        with tqdm(desc='Elements', unit_scale=1) as pbar:
            try:
                while True:
                    batches.append(sess.run(get_next))
                    pbar.update(len(batches[-1]))
            except tf.errors.OutOfRangeError:
                pass
    if batches:
        return np.concatenate(batches)
    return np.array([], dtype=ds.output_types.as_numpy_dtype)

Response: Create a single numpy array from a TF Dataset.


Instruction:
Can you tell what is the following Python 3 function doing
def _histogram_move_keys_by_game(sess, ds, batch_size=8*1024):
    ds = ds.batch(batch_size)
    # Turns 'g_0000001234_m_133' into 'g_0000001234'
    ds = ds.map(lambda x: tf.strings.substr(x, 0, 12))
    iterator = ds.make_initializable_iterator()
    sess.run(iterator.initializer)
    get_next = iterator.get_next()
    h = collections.Counter()
    try:
        while True:
            h.update(sess.run(get_next))
    except tf.errors.OutOfRangeError:
        pass
    # NOTE:  Cannot be truly sure the count is right till the end.
    return h

Response: Given dataset of key names return histogram of moves and game players.


Instruction:
Can you create a Python 3 function that
turns keys of a Bigtable dataset into an array.

Response:
def _game_keys_as_array(ds):
    """Turn keys of a Bigtable dataset into an array.

    Take g_GGG_m_MMM and create GGG.MMM numbers.

    Valuable when visualizing the distribution of a given dataset in
    the game keyspace.
    """
    ds = ds.map(lambda row_key, cell: row_key)
    # want 'g_0000001234_m_133' is '0000001234.133' and so forth
    ds = ds.map(lambda x:
                tf.strings.to_number(tf.strings.substr(x, 2, 10) +
                                     '.' +
                                     tf.strings.substr(x, 15, 3),
                                     out_type=tf.float64))
    return make_single_array(ds)


Instruction:
Write a Python 3 function that can
delete the given row keys from the given Bigtable.

Response:
def _delete_rows(args):
    """Delete the given row keys from the given Bigtable.

    The args are (BigtableSpec, row_keys), but are passed
    as a single argument in order to work with
    multiprocessing.Pool.map.  This is also the reason why this is a
    top-level function instead of a method.

    """
    btspec, row_keys = args
    bt_table = bigtable.Client(btspec.project).instance(
        btspec.instance).table(btspec.table)
    rows = [bt_table.row(k) for k in row_keys]
    for r in rows:
        r.delete()
    bt_table.mutate_rows(rows)
    return row_keys


Instruction:
Write a Python 3 script for
setting the metadata cell used to block until enough new games have been played.

Response:
def set_fresh_watermark(game_queue, count_from, window_size,
                        fresh_fraction=0.05, minimum_fresh=20000):
    """Sets the metadata cell used to block until some quantity of games have been played.

    This sets the 'freshness mark' on the `game_queue`, used to block training
    until enough new games have been played.  The number of fresh games required
    is the larger of:
       - The fraction of the total window size
       - The `minimum_fresh` parameter
    The number of games required can be indexed from the 'count_from' parameter.
    Args:
      game_queue: A GameQueue object, on whose backing table will be modified.
      count_from: the index of the game to compute the increment from
      window_size:  an integer indicating how many past games are considered
      fresh_fraction: a float in (0,1] indicating the fraction of games to wait for
      minimum_fresh:  an integer indicating the lower bound on the number of new
      games.
    """
    already_played = game_queue.latest_game_number - count_from
    print("== already_played: ", already_played, flush=True)
    if window_size > count_from:  # How to handle the case when the window is not yet 'full'
        game_queue.require_fresh_games(int(minimum_fresh * .9))
    else:
        num_to_play = max(0, math.ceil(window_size * .9 * fresh_fraction) - already_played)
        print("== Num to play: ", num_to_play, flush=True)
        game_queue.require_fresh_games(num_to_play)


Instruction:
Implement a function in Python 3 to
get a dataset of serialized TFExamples from the last n games.

Response:
def get_unparsed_moves_from_last_n_games(games, games_nr, n,
                                         moves=2**21,
                                         shuffle=True,
                                         column_family=TFEXAMPLE,
                                         column='example',
                                         values_only=True):
    """Get a dataset of serialized TFExamples from the last N games.

    Args:
      games, games_nr: GameQueues of the regular selfplay and calibration
        (aka 'no resign') games to sample from.
      n:  an integer indicating how many past games should be sourced.
      moves:  an integer indicating how many moves should be sampled
        from those N games.
      column_family:  name of the column family containing move examples.
      column:  name of the column containing move examples.
      shuffle:  if True, shuffle the selected move examples.
      values_only: if True, return only column values, no row keys.

    Returns:
      A dataset containing no more than `moves` examples, sampled
        randomly from the last `n` games in the table.
    """
    # The prefixes and suffixes below have the following meanings:
    #   ct_: count
    #   fr_: fraction
    #    _r: resign (ordinary)
    #   _nr: no-resign
    ct_r, ct_nr = 9, 1
    ct_total = ct_r + ct_nr
    fr_r = ct_r / ct_total
    fr_nr = ct_nr / ct_total
    resign = games.moves_from_last_n_games(
        math.ceil(n * fr_r),
        math.ceil(moves * fr_r),
        shuffle,
        column_family, column)
    no_resign = games_nr.moves_from_last_n_games(
        math.floor(n * fr_nr),
        math.floor(moves * fr_nr),
        shuffle,
        column_family, column)
    selection = np.array([0] * ct_r + [1] * ct_nr, dtype=np.int64)
    choice = tf.data.Dataset.from_tensor_slices(selection).repeat().take(moves)
    ds = tf.contrib.data.choose_from_datasets([resign, no_resign], choice)
    if shuffle:
        ds = ds.shuffle(len(selection) * 2)
    if values_only:
        ds = ds.map(lambda row_name, s: s)
    return ds


Instruction:
Explain what the following Python 3 code does
def count_elements_in_dataset(ds, batch_size=1*1024, parallel_batch=8):
    with tf.Session() as sess:
        dsc = ds.apply(tf.contrib.data.enumerate_dataset())
        dsc = dsc.apply(
            tf.contrib.data.map_and_batch(lambda c, v: c, batch_size,
                                          num_parallel_batches=parallel_batch))
        iterator = dsc.make_initializable_iterator()
        sess.run(iterator.initializer)
        get_next = iterator.get_next()
        counted = 0
        try:
            while True:
                # The numbers in the tensors are 0-based indicies,
                # so add 1 to get the number counted.
                counted = sess.run(tf.reduce_max(get_next)) + 1
                utils.dbg('Counted so far: %d' % counted)
        except tf.errors.OutOfRangeError:
            pass
        utils.dbg('Counted total: %d' % counted)
        return counted

Response: Count and return all the elements in the given dataset.


Instruction:
Create a Python 3 function for
creating the table underlying the queue.

Response:
def create(self):
        """Create the table underlying the queue.

        Create the 'metadata' and 'tfexample' column families
        and their properties.
        """
        if self.bt_table.exists():
            utils.dbg('Table already exists')
            return

        max_versions_rule = bigtable_column_family.MaxVersionsGCRule(1)
        self.bt_table.create(column_families={
            METADATA: max_versions_rule,
            TFEXAMPLE: max_versions_rule})


Instruction:
How would you code a function in Python 3 to
give a range of games return the games sorted by time.

Response:
def games_by_time(self, start_game, end_game):
        """Given a range of games, return the games sorted by time.

        Returns [(time, game_number), ...]

        The time will be a `datetime.datetime` and the game
        number is the integer used as the basis of the row ID.

        Note that when a cluster of self-play nodes are writing
        concurrently, the game numbers may be out of order.
        """
        move_count = b'move_count'
        rows = self.bt_table.read_rows(
            ROWCOUNT_PREFIX.format(start_game),
            ROWCOUNT_PREFIX.format(end_game),
            filter_=bigtable_row_filters.ColumnRangeFilter(
                METADATA, move_count, move_count))

        def parse(r):
            rk = str(r.row_key, 'utf-8')
            game = _game_from_counter.match(rk).groups()[0]
            return (r.cells[METADATA][move_count][0].timestamp, game)
        return sorted([parse(r) for r in rows], key=operator.itemgetter(0))


Instruction:
Can you generate the documentation for the following Python 3 function
def delete_row_range(self, format_str, start_game, end_game):
        row_keys = make_single_array(
            self.tf_table.keys_by_range_dataset(
                format_str.format(start_game),
                format_str.format(end_game)))
        row_keys = list(row_keys)
        if not row_keys:
            utils.dbg('No rows left for games %d..%d' % (
                start_game, end_game))
            return
        utils.dbg('Deleting %d rows:  %s..%s' % (
            len(row_keys), row_keys[0], row_keys[-1]))

        # Reverse the keys so that the queue is left in a more
        # sensible end state if you change your mind (say, due to a
        # mistake in the timestamp) and abort the process: there will
        # be a bit trimmed from the end, rather than a bit
        # trimmed out of the middle.
        row_keys.reverse()
        total_keys = len(row_keys)
        utils.dbg('Deleting total of %d keys' % total_keys)
        concurrency = min(MAX_BT_CONCURRENCY,
                          multiprocessing.cpu_count() * 2)
        with multiprocessing.Pool(processes=concurrency) as pool:
            batches = []
            with tqdm(desc='Keys', unit_scale=2, total=total_keys) as pbar:
                for b in utils.iter_chunks(bigtable.row.MAX_MUTATIONS,
                                           row_keys):
                    pbar.update(len(b))
                    batches.append((self.btspec, b))
                    if len(batches) >= concurrency:
                        pool.map(_delete_rows, batches)
                        batches = []
                pool.map(_delete_rows, batches)
                batches = []

Response: Delete rows related to the given game range.


Instruction:
Can you generate the documentation for the following Python 3 function
def trim_games_since(self, t, max_games=500000):
        latest = self.latest_game_number
        earliest = int(latest - max_games)
        gbt = self.games_by_time(earliest, latest)
        if not gbt:
            utils.dbg('No games between %d and %d' % (earliest, latest))
            return
        most_recent = gbt[-1]
        if isinstance(t, datetime.timedelta):
            target = most_recent[0] - t
        else:
            target = t
        i = bisect.bisect_right(gbt, (target,))
        if i >= len(gbt):
            utils.dbg('Last game is already at %s' % gbt[-1][0])
            return
        when, which = gbt[i]
        utils.dbg('Most recent:  %s  %s' % most_recent)
        utils.dbg('     Target:  %s  %s' % (when, which))
        which = int(which)
        self.delete_row_range(ROW_PREFIX, which, latest)
        self.delete_row_range(ROWCOUNT_PREFIX, which, latest)
        self.latest_game_number = which

Response: Trim off the games since the given time.


Instruction:
Write a Python 3 script to
give a range of games return the bleakest moves.

Response:
def bleakest_moves(self, start_game, end_game):
        """Given a range of games, return the bleakest moves.

        Returns a list of (game, move, q) sorted by q.
        """
        bleak = b'bleakest_q'
        rows = self.bt_table.read_rows(
            ROW_PREFIX.format(start_game),
            ROW_PREFIX.format(end_game),
            filter_=bigtable_row_filters.ColumnRangeFilter(
                METADATA, bleak, bleak))

        def parse(r):
            rk = str(r.row_key, 'utf-8')
            g, m = _game_row_key.match(rk).groups()
            q = r.cell_value(METADATA, bleak)
            return int(g), int(m), float(q)
        return sorted([parse(r) for r in rows], key=operator.itemgetter(2))


Instruction:
Write a Python 3 function for
requiring a given number of fresh games to be played. Args: number_fresh: integer, number of new fresh games needed Increments the cell `table_state=metadata:wait_for_game_number` by the given number of games. This will cause `self.wait_for_fresh_games()` to block until the game counter has reached this number.

Response:
def require_fresh_games(self, number_fresh):
        """Require a given number of fresh games to be played.

        Args:
          number_fresh:  integer, number of new fresh games needed

        Increments the cell `table_state=metadata:wait_for_game_number`
        by the given number of games.  This will cause
        `self.wait_for_fresh_games()` to block until the game
        counter has reached this number.
        """
        latest = self.latest_game_number
        table_state = self.bt_table.row(TABLE_STATE)
        table_state.set_cell(METADATA, WAIT_CELL, int(latest + number_fresh))
        table_state.commit()
        print("== Setting wait cell to ", int(latest + number_fresh), flush=True)


Instruction:
How would you explain what the following Python 3 function does
def wait_for_fresh_games(self, poll_interval=15.0):
        wait_until_game = self.read_wait_cell()
        if not wait_until_game:
            return
        latest_game = self.latest_game_number
        last_latest = latest_game
        while latest_game < wait_until_game:
            utils.dbg('Latest game {} not yet at required game {} '
                      '(+{}, {:0.3f} games/sec)'.format(
                          latest_game,
                          wait_until_game,
                          latest_game - last_latest,
                          (latest_game - last_latest) / poll_interval
                      ))
            time.sleep(poll_interval)
            last_latest = latest_game
            latest_game = self.latest_game_number

Response: Block the caller until required new games have been played.


Instruction:
Implement a function in Python 3 to
read the value of the cell holding the wait value

Response:
def read_wait_cell(self):
        """Read the value of the cell holding the 'wait' value,

        Returns the int value of whatever it has, or None if the cell doesn't
        exist.
        """

        table_state = self.bt_table.read_row(
            TABLE_STATE,
            filter_=bigtable_row_filters.ColumnRangeFilter(
                METADATA, WAIT_CELL, WAIT_CELL))
        if table_state is None:
            utils.dbg('No waiting for new games needed; '
                      'wait_for_game_number column not in table_state')
            return None
        value = table_state.cell_value(METADATA, WAIT_CELL)
        if not value:
            utils.dbg('No waiting for new games needed; '
                      'no value in wait_for_game_number cell '
                      'in table_state')
            return None
        return cbt_intvalue(value)


Instruction:
Implement a Python 3 function for
counting the total number of moves in a game range.

Response:
def count_moves_in_game_range(self, game_begin, game_end):
        """Count the total moves in a game range.

        Args:
          game_begin:  integer, starting game
          game_end:  integer, ending game

        Uses the `ct_` keyspace for rapid move summary.
        """
        rows = self.bt_table.read_rows(
            ROWCOUNT_PREFIX.format(game_begin),
            ROWCOUNT_PREFIX.format(game_end),
            filter_=bigtable_row_filters.ColumnRangeFilter(
                METADATA, MOVE_COUNT, MOVE_COUNT))
        return sum([int(r.cell_value(METADATA, MOVE_COUNT)) for r in rows])


Instruction:
How would you implement a function in Python 3 that
returns a dataset of samples and shuffled moves from the given game range.

Response:
def moves_from_games(self, start_game, end_game, moves, shuffle,
                         column_family, column):
        """Dataset of samples and/or shuffled moves from game range.

        Args:
          n:  an integer indicating how many past games should be sourced.
          moves:  an integer indicating how many moves should be sampled
            from those N games.
          column_family:  name of the column family containing move examples.
          column:  name of the column containing move examples.
          shuffle:  if True, shuffle the selected move examples.

        Returns:
          A dataset containing no more than `moves` examples, sampled
            randomly from the last `n` games in the table.
        """
        start_row = ROW_PREFIX.format(start_game)
        end_row = ROW_PREFIX.format(end_game)
        # NOTE:  Choose a probability high enough to guarantee at least the
        # required number of moves, by using a slightly lower estimate
        # of the total moves, then trimming the result.
        total_moves = self.count_moves_in_game_range(start_game, end_game)
        probability = moves / (total_moves * 0.99)
        utils.dbg('Row range: %s - %s; total moves: %d; probability %.3f; moves %d' % (
            start_row, end_row, total_moves, probability, moves))
        ds = self.tf_table.parallel_scan_range(start_row, end_row,
                                               probability=probability,
                                               columns=[(column_family, column)])
        if shuffle:
            utils.dbg('Doing a complete shuffle of %d moves' % moves)
            ds = ds.shuffle(moves)
        ds = ds.take(moves)
        return ds


Instruction:
Make a summary of the following Python 3 code
def moves_from_last_n_games(self, n, moves, shuffle,
                                column_family, column):
        """Randomly choose a given number of moves from the last n games.

        Args:
          n:  number of games at the end of this GameQueue to source.
          moves:  number of moves to be sampled from `n` games.
          shuffle:  if True, shuffle the selected moves.
          column_family:  name of the column family containing move examples.
          column:  name of the column containing move examples.

        Returns:
          a dataset containing the selected moves.
        """
        self.wait_for_fresh_games()
        latest_game = self.latest_game_number
        utils.dbg('Latest game in %s: %s' % (self.btspec.table, latest_game))
        if latest_game == 0:
            raise ValueError('Cannot find a latest game in the table')

        start = int(max(0, latest_game - n))
        ds = self.moves_from_games(start, latest_game, moves, shuffle,
                                   column_family, column)
        return ds

Response: Randomly choose a given number of moves from the last n games.


Instruction:
How would you explain what the following Python 3 function does
def _write_move_counts(self, sess, h):
        def gen():
            for k, v in h.items():
                # The keys in the histogram may be of type 'bytes'
                k = str(k, 'utf-8')
                vs = str(v)
                yield (k.replace('g_', 'ct_') + '_%d' % v, vs)
                yield (k + '_m_000', vs)
        mc = tf.data.Dataset.from_generator(gen, (tf.string, tf.string))
        wr_op = self.tf_table.write(mc,
                                    column_families=[METADATA],
                                    columns=[MOVE_COUNT])
        sess.run(wr_op)

Response: Add move counts from the given histogram to the table.


Instruction:
Can you implement a function in Python 3 that
uses to update the move_count cells for older games.

Response:
def update_move_counts(self, start_game, end_game, interval=1000):
        """Used to update the move_count cell for older games.

        Should not be needed except for backfill or repair.

        move_count cells will be updated in both g_<game_id>_m_000 rows
        and ct_<game_id>_<move_count> rows.
        """
        for g in range(start_game, end_game, interval):
            with tf.Session() as sess:
                start_row = ROW_PREFIX.format(g)
                end_row = ROW_PREFIX.format(g + interval)
                print('Range:', start_row, end_row)
                start_time = time.time()
                ds = self.tf_table.keys_by_range_dataset(start_row, end_row)
                h = _histogram_move_keys_by_game(sess, ds)
                self._write_move_counts(sess, h)
                end_time = time.time()
                elapsed = end_time - start_time
                print('  games/sec:', len(h)/elapsed)


Instruction:
Given the following Python 3 function, write the documentation
def get_filenames(is_training, data_dir):
  if is_training:
    return [
        os.path.join(data_dir, 'train-%05d-of-01024' % i)
        for i in range(_NUM_TRAIN_FILES)]
  else:
    return [
        os.path.join(data_dir, 'validation-%05d-of-00128' % i)
        for i in range(128)]

Response: Return filenames for dataset.


Instruction:
Can you write a function in Python 3 where it
parses an example proto containing a training example of an image.

Response:
def _parse_example_proto(example_serialized):
  """Parses an Example proto containing a training example of an image.

  The output of the build_image_data.py image preprocessing script is a dataset
  containing serialized Example protocol buffers. Each Example proto contains
  the following fields (values are included as examples):

    image/height: 462
    image/width: 581
    image/colorspace: 'RGB'
    image/channels: 3
    image/class/label: 615
    image/class/synset: 'n03623198'
    image/class/text: 'knee pad'
    image/object/bbox/xmin: 0.1
    image/object/bbox/xmax: 0.9
    image/object/bbox/ymin: 0.2
    image/object/bbox/ymax: 0.6
    image/object/bbox/label: 615
    image/format: 'JPEG'
    image/filename: 'ILSVRC2012_val_00041207.JPEG'
    image/encoded: <JPEG encoded string>

  Args:
    example_serialized: scalar Tensor tf.string containing a serialized
      Example protocol buffer.

  Returns:
    image_buffer: Tensor tf.string containing the contents of a JPEG file.
    label: Tensor tf.int32 containing the label.
    bbox: 3-D float Tensor of bounding boxes arranged [1, num_boxes, coords]
      where each coordinate is [0, 1) and the coordinates are arranged as
      [ymin, xmin, ymax, xmax].
  """
  # Dense features in Example proto.
  feature_map = {
      'image/encoded': tf.FixedLenFeature([], dtype=tf.string,
                                          default_value=''),
      'image/class/label': tf.FixedLenFeature([], dtype=tf.int64,
                                              default_value=-1),
      'image/class/text': tf.FixedLenFeature([], dtype=tf.string,
                                             default_value=''),
  }
  sparse_float32 = tf.VarLenFeature(dtype=tf.float32)
  # Sparse features in Example proto.
  feature_map.update(
      {k: sparse_float32 for k in ['image/object/bbox/xmin',
                                   'image/object/bbox/ymin',
                                   'image/object/bbox/xmax',
                                   'image/object/bbox/ymax']})

  features = tf.parse_single_example(example_serialized, feature_map)
  label = tf.cast(features['image/class/label'], dtype=tf.int32)

  return features['image/encoded'], label


Instruction:
Can you generate the documentation for the following Python 3 function
def parse_record(raw_record, is_training, dtype):
  image_buffer, label = _parse_example_proto(raw_record)

  image = imagenet_preprocessing.preprocess_image(
      image_buffer=image_buffer,
      output_height=_DEFAULT_IMAGE_SIZE,
      output_width=_DEFAULT_IMAGE_SIZE,
      num_channels=_NUM_CHANNELS,
      is_training=is_training)
  image = tf.cast(image, dtype)

  return image, label

Response: Parses a record containing a training example of an image.


Instruction:
Can you tell what is the following Python 3 function doing
def input_fn(is_training, data_dir, batch_size, num_epochs=1, num_gpus=None,
             dtype=tf.float32):
  """Input function which provides batches for train or eval.

  Args:
    is_training: A boolean denoting whether the input is for training.
    data_dir: The directory containing the input data.
    batch_size: The number of samples per batch.
    num_epochs: The number of epochs to repeat the dataset.
    num_gpus: The number of gpus used for training.
    dtype: Data type to use for images/features

  Returns:
    A dataset that can be used for iteration.
  """
  mlperf_log.resnet_print(key=mlperf_log.INPUT_ORDER)
  filenames = get_filenames(is_training, data_dir)
  dataset = tf.data.Dataset.from_tensor_slices(filenames)

  if is_training:
    # Shuffle the input files
    dataset = dataset.shuffle(buffer_size=_NUM_TRAIN_FILES)

  # Convert to individual records
  dataset = dataset.flat_map(tf.data.TFRecordDataset)

  return resnet_run_loop.process_record_dataset(
      dataset=dataset,
      is_training=is_training,
      batch_size=batch_size,
      shuffle_buffer=_SHUFFLE_BUFFER,
      parse_record_fn=parse_record,
      num_epochs=num_epochs,
      num_gpus=num_gpus,
      examples_per_epoch=_NUM_IMAGES['train'] if is_training else None,
      dtype=dtype
  )

Response: Input function which provides batches for training or evaluation.


Instruction:
Write a Python 3 function for
retrieving the size of each block_layer in the ResNet model. The number of block layers used for the Resnet model varies according to the size of the model. This helper grabs the layer set we want, throwing an error if a non-standard size has been selected. Args: resnet_size: The number of convolutional layers needed in the model. Returns: A list of block sizes to use in building the model. Raises: KeyError: if invalid resnet_size is received.

Response:
def _get_block_sizes(resnet_size):
  """Retrieve the size of each block_layer in the ResNet model.

  The number of block layers used for the Resnet model varies according
  to the size of the model. This helper grabs the layer set we want, throwing
  an error if a non-standard size has been selected.

  Args:
    resnet_size: The number of convolutional layers needed in the model.

  Returns:
    A list of block sizes to use in building the model.

  Raises:
    KeyError: if invalid resnet_size is received.
  """
  choices = {
      18: [2, 2, 2, 2],
      34: [3, 4, 6, 3],
      50: [3, 4, 6, 3],
      101: [3, 4, 23, 3],
      152: [3, 8, 36, 3],
      200: [3, 24, 36, 3]
  }

  try:
    return choices[resnet_size]
  except KeyError:
    err = ('Could not find layers for selected Resnet size.\n'
           'Size received: {}; sizes allowed: {}.'.format(
               resnet_size, choices.keys()))
    raise ValueError(err)


Instruction:
How would you code a function in Python 3 to
wrap for MLPerf compliance logging calls.

Response:
def gnmt_print(*args, **kwargs):
    """
    Wrapper for MLPerf compliance logging calls.
    All arguments but 'sync' are passed to mlperf_log.gnmt_print function.
    If 'sync' is set to True then the wrapper will synchronize all distributed
    workers. 'sync' should be set to True for all compliance tags that require
    accurate timing (RUN_START, RUN_STOP etc.)
    """
    if kwargs.pop('sync'):
        barrier()
    if get_rank() == 0:
        kwargs['stack_offset'] = 2
        mlperf_log.gnmt_print(*args, **kwargs)


Instruction:
Implement a Python 3 function for
initializing weights and biases of the PyTorch LSTM layer.

Response:
def init_lstm_(lstm, init_weight=0.1):
    """
    Initializes weights of LSTM layer.
    Weights and biases are initialized with uniform(-init_weight, init_weight)
    distribution.

    :param lstm: instance of torch.nn.LSTM
    :param init_weight: range for the uniform initializer
    """
    # Initialize hidden-hidden weights
    init.uniform_(lstm.weight_hh_l0.data, -init_weight, init_weight)
    # Initialize input-hidden weights:
    init.uniform_(lstm.weight_ih_l0.data, -init_weight, init_weight)

    # Initialize bias. PyTorch LSTM has two biases, one for input-hidden GEMM
    # and the other for hidden-hidden GEMM. Here input-hidden bias is
    # initialized with uniform distribution and hidden-hidden bias is
    # initialized with zeros.
    init.uniform_(lstm.bias_ih_l0.data, -init_weight, init_weight)
    init.zeros_(lstm.bias_hh_l0.data)

    if lstm.bidirectional:
        init.uniform_(lstm.weight_hh_l0_reverse.data, -init_weight, init_weight)
        init.uniform_(lstm.weight_ih_l0_reverse.data, -init_weight, init_weight)

        init.uniform_(lstm.bias_ih_l0_reverse.data, -init_weight, init_weight)
        init.zeros_(lstm.bias_hh_l0_reverse.data)


Instruction:
Given the following Python 3 function, write the documentation
def setup_seeds(master_seed, epochs, device):
    if master_seed is None:
        # random master seed, random.SystemRandom() uses /dev/urandom on Unix
        master_seed = random.SystemRandom().randint(0, 2**32 - 1)
        if get_rank() == 0:
            # master seed is reported only from rank=0 worker, it's to avoid
            # confusion, seeds from rank=0 are later broadcasted to other
            # workers
            logging.info(f'Using random master seed: {master_seed}')
    else:
        # master seed was specified from command line
        logging.info(f'Using master seed from command line: {master_seed}')

    gnmt_print(key=mlperf_log.RUN_SET_RANDOM_SEED, value=master_seed,
               sync=False)

    # initialize seeding RNG
    seeding_rng = random.Random(master_seed)

    # generate worker seeds, one seed for every distributed worker
    worker_seeds = generate_seeds(seeding_rng, get_world_size())

    # generate seeds for data shuffling, one seed for every epoch
    shuffling_seeds = generate_seeds(seeding_rng, epochs)

    # broadcast seeds from rank=0 to other workers
    worker_seeds = broadcast_seeds(worker_seeds, device)
    shuffling_seeds = broadcast_seeds(shuffling_seeds, device)
    return worker_seeds, shuffling_seeds

Response: Function creates seeds from one master_seed.


Instruction:
Write a Python 3 function that can
work as a temporary distributed barrier, currently pytorch doesn't implement barrier for NCCL backend. Calls all_reduce on dummy tensor and synchronizes with GPU.

Response:
def barrier():
    """
    Works as a temporary distributed barrier, currently pytorch
    doesn't implement barrier for NCCL backend.
    Calls all_reduce on dummy tensor and synchronizes with GPU.
    """
    if torch.distributed.is_available() and torch.distributed.is_initialized():
        torch.distributed.all_reduce(torch.cuda.FloatTensor(1))
        torch.cuda.synchronize()


Instruction:
Can you write a function in Python 3 where it
gets distributed rank or returns zero if distributed is not initialized

Response:
def get_rank():
    """
    Gets distributed rank or returns zero if distributed is not initialized.
    """
    if torch.distributed.is_available() and torch.distributed.is_initialized():
        rank = torch.distributed.get_rank()
    else:
        rank = 0
    return rank


Instruction:
Can you generate a brief explanation for the following Python 3 code
def get_world_size():
    if torch.distributed.is_available() and torch.distributed.is_initialized():
        world_size = torch.distributed.get_world_size()
    else:
        world_size = 1
    return world_size

Response: Gets total number of workers or returns one if distributed is not initialized


Instruction:
Implement a Python 3 function for
configuring logging. By default logs from all workers are printed to the console, entries are prefixed with "N: " where N is the rank of the worker. Logs printed to the console don't include timestaps. Full logs with timestamps are saved to the log_file file.

Response:
def setup_logging(log_file=os.devnull):
    """
    Configures logging.
    By default logs from all workers are printed to the console, entries are
    prefixed with "N: " where N is the rank of the worker. Logs printed to the
    console don't include timestaps.
    Full logs with timestamps are saved to the log_file file.
    """
    class RankFilter(logging.Filter):
        def __init__(self, rank):
            self.rank = rank

        def filter(self, record):
            record.rank = self.rank
            return True

    rank = get_rank()
    rank_filter = RankFilter(rank)

    logging_format = "%(asctime)s - %(levelname)s - %(rank)s - %(message)s"
    logging.basicConfig(level=logging.DEBUG,
                        format=logging_format,
                        datefmt="%Y-%m-%d %H:%M:%S",
                        filename=log_file,
                        filemode='w')
    console = logging.StreamHandler(sys.stdout)
    console.setLevel(logging.INFO)
    formatter = logging.Formatter('%(rank)s: %(message)s')
    console.setFormatter(formatter)
    logging.getLogger('').addHandler(console)
    logging.getLogger('').addFilter(rank_filter)


Instruction:
Create a Python 3 function to
set the device based on local_rank and returns instance of torch. device.

Response:
def set_device(cuda, local_rank):
    """
    Sets device based on local_rank and returns instance of torch.device.

    :param cuda: if True: use cuda
    :param local_rank: local rank of the worker
    """
    if cuda:
        torch.cuda.set_device(local_rank)
        device = torch.device('cuda')
    else:
        device = torch.device('cpu')
    return device


Instruction:
Can you create a Python 3 function that
initializes distributed backend. :param cuda: (bool) if True initializes nccl backend, if False initializes gloo backend

Response:
def init_distributed(cuda):
    """
    Initializes distributed backend.

    :param cuda: (bool) if True initializes nccl backend, if False initializes
        gloo backend
    """
    world_size = int(os.environ.get('WORLD_SIZE', 1))
    distributed = (world_size > 1)
    if distributed:
        backend = 'nccl' if cuda else 'gloo'
        dist.init_process_group(backend=backend,
                                init_method='env://')
        assert dist.is_initialized()
    return distributed


Instruction:
Implement a Python 3 function for
printing information about execution environment.

Response:
def log_env_info():
    """
    Prints information about execution environment.
    """
    logging.info('Collecting environment information...')
    env_info = torch.utils.collect_env.get_pretty_env_info()
    logging.info(f'{env_info}')


Instruction:
Write a Python 3 function for
reducing the average value over all workers.

Response:
def reduce(self, op):
        """
        Reduces average value over all workers.

        :param op: 'sum' or 'mean', reduction operator
        """
        if op not in ('sum', 'mean'):
            raise NotImplementedError

        distributed = (get_world_size() > 1)
        if distributed:
            # Backward/forward compatibility around
            # https://github.com/pytorch/pytorch/commit/540ef9b1fc5506369a48491af8a285a686689b36 and
            # https://github.com/pytorch/pytorch/commit/044d00516ccd6572c0d6ab6d54587155b02a3b86
            # To accomodate change in Pytorch's distributed API
            if hasattr(dist, "get_backend"):
                _backend = dist.get_backend()
                if hasattr(dist, "DistBackend"):
                    backend_enum_holder = dist.DistBackend
                else:
                    backend_enum_holder = dist.Backend
            else:
                _backend = dist._backend
                backend_enum_holder = dist.dist_backend

            cuda = _backend == backend_enum_holder.NCCL

            if cuda:
                avg = torch.cuda.FloatTensor([self.avg])
                _sum = torch.cuda.FloatTensor([self.sum])
            else:
                avg = torch.FloatTensor([self.avg])
                _sum = torch.FloatTensor([self.sum])

            _reduce_op = dist.reduce_op if hasattr(dist, "reduce_op") else dist.ReduceOp
            dist.all_reduce(avg, op=_reduce_op.SUM)
            dist.all_reduce(_sum, op=_reduce_op.SUM)
            self.avg = avg.item()
            self.sum = _sum.item()

            if op == 'mean':
                self.avg /= get_world_size()
                self.sum /= get_world_size()


Instruction:
How would you explain what the following Python 3 function does
def build_dataset(dataset_list, transforms, dataset_catalog, is_train=True):
    if not isinstance(dataset_list, (list, tuple)):
        raise RuntimeError(
            "dataset_list should be a list of strings, got {}".format(dataset_list)
        )
    datasets = []
    total_datasets_size = 0
    for dataset_name in dataset_list:
        data = dataset_catalog.get(dataset_name)
        factory = getattr(D, data["factory"])
        args = data["args"]
        # for COCODataset, we want to remove images without annotations
        # during training
        if data["factory"] == "COCODataset":
            args["remove_images_without_annotations"] = is_train
        if data["factory"] == "PascalVOCDataset":
            args["use_difficult"] = not is_train
        args["transforms"] = transforms
        # make dataset from factory
        dataset = factory(**args)
        total_datasets_size += len(dataset)
        datasets.append(dataset)

    # for testing, return a list of datasets
    if not is_train:
        return datasets, total_datasets_size

    # for training, concatenate all datasets into a single one
    dataset = datasets[0]
    if len(datasets) > 1:
        dataset = D.ConcatDataset(datasets)

    return [dataset], total_datasets_size

Response: Builds a single dataset from a list of datasets.


Instruction:
Implement a function in Python 3 to
convert a list of Tensors to a list of ImageLists.

Response:
def to_image_list(tensors, size_divisible=0):
    """
    tensors can be an ImageList, a torch.Tensor or
    an iterable of Tensors. It can't be a numpy array.
    When tensors is an iterable of Tensors, it pads
    the Tensors with zeros so that they have the same
    shape
    """
    if isinstance(tensors, torch.Tensor) and size_divisible > 0:
        tensors = [tensors]

    if isinstance(tensors, ImageList):
        return tensors
    elif isinstance(tensors, torch.Tensor):
        # single tensor shape can be inferred
        assert tensors.dim() == 4
        image_sizes = [tensor.shape[-2:] for tensor in tensors]
        return ImageList(tensors, image_sizes)
    elif isinstance(tensors, (tuple, list)):
        max_size = tuple(max(s) for s in zip(*[img.shape for img in tensors]))

        # TODO Ideally, just remove this and let me model handle arbitrary
        # input sizs
        if size_divisible > 0:
            import math

            stride = size_divisible
            max_size = list(max_size)
            max_size[1] = int(math.ceil(max_size[1] / stride) * stride)
            max_size[2] = int(math.ceil(max_size[2] / stride) * stride)
            max_size = tuple(max_size)

        batch_shape = (len(tensors),) + max_size
        batched_imgs = tensors[0].new(*batch_shape).zero_()
        for img, pad_img in zip(tensors, batched_imgs):
            pad_img[: img.shape[0], : img.shape[1], : img.shape[2]].copy_(img)

        image_sizes = [im.shape[-2:] for im in tensors]

        return ImageList(batched_imgs, image_sizes)
    else:
        raise TypeError("Unsupported type for to_image_list: {}".format(type(tensors)))


Instruction:
Can you tell what is the following Python 3 function doing
def forward(self, features, proposals, targets=None):
        if self.training:
            with torch.no_grad():
                proposals = self.loss_evaluator.subsample(proposals, targets)

        x = self.feature_extractor(features, proposals)
        kp_logits = self.predictor(x)

        if not self.training:
            result = self.post_processor(kp_logits, proposals)
            return x, result, {}

        loss_kp = self.loss_evaluator(proposals, kp_logits)

        return x, proposals, dict(loss_kp=loss_kp)

Response: Forward method for the base class.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def get_inference_input():
    return (tf.placeholder(tf.float32,
                           [None, go.N, go.N, features_lib.NEW_FEATURES_PLANES],
                           name='pos_tensor'),
            {'pi_tensor': tf.placeholder(tf.float32, [None, go.N * go.N + 1]),
             'value_tensor': tf.placeholder(tf.float32, [None])})

Response: Set up placeholders for input features and labels.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def model_fn(features, labels, mode, params):

    policy_output, value_output, logits = model_inference_fn(
        features, mode == tf.estimator.ModeKeys.TRAIN, params)

    # train ops
    policy_cost = tf.reduce_mean(
        tf.nn.softmax_cross_entropy_with_logits_v2(
            logits=logits, labels=tf.stop_gradient(labels['pi_tensor'])))

    value_cost = params['value_cost_weight'] * tf.reduce_mean(
        tf.square(value_output - labels['value_tensor']))

    reg_vars = [v for v in tf.trainable_variables()
                if 'bias' not in v.name and 'beta' not in v.name]
    l2_cost = params['l2_strength'] * \
        tf.add_n([tf.nn.l2_loss(v) for v in reg_vars])

    combined_cost = policy_cost + value_cost + l2_cost

    global_step = tf.train.get_or_create_global_step()
    learning_rate = tf.train.piecewise_constant(
        global_step, params['lr_boundaries'], params['lr_rates'])
    update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)

    # Insert quantization ops if requested
    if params['quantize']:
        if mode == tf.estimator.ModeKeys.TRAIN:
            tf.contrib.quantize.create_training_graph(
                quant_delay=params['quant_delay'])
        else:
            tf.contrib.quantize.create_eval_graph()

    optimizer = tf.train.MomentumOptimizer(
        learning_rate, params['sgd_momentum'])
    if params['use_tpu']:
        optimizer = tpu_optimizer.CrossShardOptimizer(optimizer)
    with tf.control_dependencies(update_ops):
        train_op = optimizer.minimize(combined_cost, global_step=global_step)

    # Computations to be executed on CPU, outside of the main TPU queues.
    def eval_metrics_host_call_fn(policy_output, value_output, pi_tensor, policy_cost,
                                  value_cost, l2_cost, combined_cost, step,
                                  est_mode=tf.estimator.ModeKeys.TRAIN):
        policy_entropy = -tf.reduce_mean(tf.reduce_sum(
            policy_output * tf.log(policy_output), axis=1))
        # pi_tensor is one_hot when generated from sgfs (for supervised learning)
        # and soft-max when using self-play records. argmax normalizes the two.
        policy_target_top_1 = tf.argmax(pi_tensor, axis=1)

        policy_output_in_top1 = tf.to_float(
            tf.nn.in_top_k(policy_output, policy_target_top_1, k=1))
        policy_output_in_top3 = tf.to_float(
            tf.nn.in_top_k(policy_output, policy_target_top_1, k=3))

        policy_top_1_confidence = tf.reduce_max(policy_output, axis=1)
        policy_target_top_1_confidence = tf.boolean_mask(
            policy_output,
            tf.one_hot(policy_target_top_1, tf.shape(policy_output)[1]))

        value_cost_normalized = value_cost / params['value_cost_weight']

        with tf.variable_scope("metrics"):
            metric_ops = {
                'policy_cost': tf.metrics.mean(policy_cost),
                'value_cost': tf.metrics.mean(value_cost),
                'value_cost_normalized': tf.metrics.mean(value_cost_normalized),
                'l2_cost': tf.metrics.mean(l2_cost),
                'policy_entropy': tf.metrics.mean(policy_entropy),
                'combined_cost': tf.metrics.mean(combined_cost),

                'policy_accuracy_top_1': tf.metrics.mean(policy_output_in_top1),
                'policy_accuracy_top_3': tf.metrics.mean(policy_output_in_top3),
                'policy_top_1_confidence': tf.metrics.mean(policy_top_1_confidence),
                'policy_target_top_1_confidence': tf.metrics.mean(
                    policy_target_top_1_confidence),
                'value_confidence': tf.metrics.mean(tf.abs(value_output)),
            }

        if est_mode == tf.estimator.ModeKeys.EVAL:
            return metric_ops

        # NOTE: global_step is rounded to a multiple of FLAGS.summary_steps.
        eval_step = tf.reduce_min(step)

        # Create summary ops so that they show up in SUMMARIES collection
        # That way, they get logged automatically during training
        summary_writer = summary.create_file_writer(FLAGS.work_dir)
        with summary_writer.as_default(), \
                summary.record_summaries_every_n_global_steps(
                    params['summary_steps'], eval_step):
            for metric_name, metric_op in metric_ops.items():
                summary.scalar(metric_name, metric_op[1], step=eval_step)

        # Reset metrics occasionally so that they are mean of recent batches.
        reset_op = tf.variables_initializer(tf.local_variables("metrics"))
        cond_reset_op = tf.cond(
            tf.equal(eval_step % params['summary_steps'], tf.to_int64(1)),
            lambda: reset_op,
            lambda: tf.no_op())

        return summary.all_summary_ops() + [cond_reset_op]

    metric_args = [
        policy_output,
        value_output,
        labels['pi_tensor'],
        tf.reshape(policy_cost, [1]),
        tf.reshape(value_cost, [1]),
        tf.reshape(l2_cost, [1]),
        tf.reshape(combined_cost, [1]),
        tf.reshape(global_step, [1]),
    ]

    predictions = {
        'policy_output': policy_output,
        'value_output': value_output,
    }

    eval_metrics_only_fn = functools.partial(
        eval_metrics_host_call_fn, est_mode=tf.estimator.ModeKeys.EVAL)
    host_call_fn = functools.partial(
        eval_metrics_host_call_fn, est_mode=tf.estimator.ModeKeys.TRAIN)

    tpu_estimator_spec = tpu_estimator.TPUEstimatorSpec(
        mode=mode,
        predictions=predictions,
        loss=combined_cost,
        train_op=train_op,
        eval_metrics=(eval_metrics_only_fn, metric_args),
        host_call=(host_call_fn, metric_args)
    )
    if params['use_tpu']:
        return tpu_estimator_spec
    else:
        return tpu_estimator_spec.as_estimator_spec()

Response: Create the model for an estimator API.


Instruction:
Create a Python 3 function to
build the inference part of the model graph.

Response:
def model_inference_fn(features, training, params):
    """Builds just the inference part of the model graph.

    Args:
        features: input features tensor.
        training: True if the model is training.
        params: A dictionary

    Returns:
        (policy_output, value_output, logits) tuple of tensors.
    """

    mg_batchn = functools.partial(
        tf.layers.batch_normalization,
        axis=-1,
        momentum=.95,
        epsilon=1e-5,
        center=True,
        scale=True,
        fused=True,
        training=training)

    mg_conv2d = functools.partial(
        tf.layers.conv2d,
        filters=params['conv_width'],
        kernel_size=3,
        padding="same",
        data_format="channels_last",
        use_bias=False)

    mg_global_avgpool2d = functools.partial(
        tf.layers.average_pooling2d,
        pool_size=go.N,
        strides=1,
        padding="valid",
        data_format="channels_last")

    def mg_activation(inputs):
        if FLAGS.use_swish:
            return tf.nn.swish(inputs)

        return tf.nn.relu(inputs)

    def residual_inner(inputs):
        conv_layer1 = mg_batchn(mg_conv2d(inputs))
        initial_output = mg_activation(conv_layer1)
        conv_layer2 = mg_batchn(mg_conv2d(initial_output))
        return conv_layer2

    def mg_res_layer(inputs):
        residual = residual_inner(inputs)
        output = mg_activation(inputs + residual)
        return output

    def mg_squeeze_excitation_layer(inputs):
        # Hu, J., Shen, L., & Sun, G. (2018). Squeeze-and-Excitation Networks.
        # 2018 IEEE/CVF Conference on Computer Vision, 7132-7141.
        # arXiv:1709.01507 [cs.CV]

        channels = params['conv_width']
        ratio = FLAGS.SE_ratio
        assert channels % ratio == 0

        residual = residual_inner(inputs)
        pool = mg_global_avgpool2d(residual)
        fc1 = tf.layers.dense(pool, units=channels // ratio)
        squeeze = mg_activation(fc1)

        if FLAGS.use_SE_bias:
            fc2 = tf.layers.dense(squeeze, units=2*channels)
            # Channels_last so axis = 3 = -1
            gamma, bias = tf.split(fc2, 2, axis=3)
        else:
            gamma = tf.layers.dense(squeeze, units=channels)
            bias = 0

        sig = tf.nn.sigmoid(gamma)
        # Explicitly signal the broadcast.
        scale = tf.reshape(sig, [-1, 1, 1, channels])

        excitation = tf.multiply(scale, residual) + bias
        return mg_activation(inputs + excitation)

    initial_block = mg_activation(mg_batchn(mg_conv2d(features)))

    # the shared stack
    shared_output = initial_block
    for _ in range(params['trunk_layers']):
        if FLAGS.use_SE or FLAGS.use_SE_bias:
            shared_output = mg_squeeze_excitation_layer(shared_output)
        else:
            shared_output = mg_res_layer(shared_output)

    # Policy head
    policy_conv = mg_conv2d(
        shared_output, filters=params['policy_conv_width'], kernel_size=1)
    policy_conv = mg_activation(mg_batchn(policy_conv, center=False, scale=False))
    logits = tf.layers.dense(
        tf.reshape(
            policy_conv, [-1, params['policy_conv_width'] * go.N * go.N]),
        go.N * go.N + 1)

    policy_output = tf.nn.softmax(logits, name='policy_output')

    # Value head
    value_conv = mg_conv2d(
        shared_output, filters=params['value_conv_width'], kernel_size=1)
    value_conv = mg_activation(mg_batchn(value_conv, center=False, scale=False))

    value_fc_hidden = mg_activation(tf.layers.dense(
        tf.reshape(value_conv, [-1, params['value_conv_width'] * go.N * go.N]),
        params['fc_width']))
    value_output = tf.nn.tanh(
        tf.reshape(tf.layers.dense(value_fc_hidden, 1), [-1]),
        name='value_output')

    return policy_output, value_output, logits


Instruction:
Create a Python 3 function to
build the model graph suitable for running on TPU.

Response:
def tpu_model_inference_fn(features):
    """Builds the model graph suitable for running on TPU.

    It does two things:
     1) Mark all weights as constant, which improves TPU inference performance
        because it prevents the weights being transferred to the TPU every call
        to Session.run().
     2) Adds constant to the graph with a unique value and marks it as a
        dependency on the rest of the model. This works around a TensorFlow bug
        that prevents multiple models being run on a single TPU.

    Returns:
        (policy_output, value_output, logits) tuple of tensors.
    """
    def custom_getter(getter, name, *args, **kwargs):
        with tf.control_dependencies(None):
            return tf.guarantee_const(
                getter(name, *args, **kwargs), name=name + "/GuaranteeConst")
    with tf.variable_scope("", custom_getter=custom_getter):
        # TODO(tommadams): remove the tf.control_dependencies context manager
        # when a fixed version of TensorFlow is released.
        t = int(time.time())
        epoch_time = tf.constant(t, name='epoch_time_%d' % t)
        with tf.control_dependencies([epoch_time]):
            return model_inference_fn(features, False, FLAGS.flag_values_dict())


Instruction:
Can you implement a function in Python 3 that
initializes a tf. Estimator run with random initial weights.

Response:
def bootstrap():
    """Initialize a tf.Estimator run with random initial weights."""
    # a bit hacky - forge an initial checkpoint with the name that subsequent
    # Estimator runs will expect to find.
    #
    # Estimator will do this automatically when you call train(), but calling
    # train() requires data, and I didn't feel like creating training data in
    # order to run the full train pipeline for 1 step.
    maybe_set_seed()
    initial_checkpoint_name = 'model.ckpt-1'
    save_file = os.path.join(FLAGS.work_dir, initial_checkpoint_name)
    sess = tf.Session(graph=tf.Graph())
    with sess.graph.as_default():
        features, labels = get_inference_input()
        model_fn(features, labels, tf.estimator.ModeKeys.PREDICT,
                 params=FLAGS.flag_values_dict())
        sess.run(tf.global_variables_initializer())
        tf.train.Saver().save(sess, save_file)


Instruction:
How would you explain what the following Python 3 function does
def export_model(model_path):
    estimator = tf.estimator.Estimator(model_fn, model_dir=FLAGS.work_dir,
                                       params=FLAGS.flag_values_dict())
    latest_checkpoint = estimator.latest_checkpoint()
    all_checkpoint_files = tf.gfile.Glob(latest_checkpoint + '*')
    for filename in all_checkpoint_files:
        suffix = filename.partition(latest_checkpoint)[2]
        destination_path = model_path + suffix
        print("Copying {} to {}".format(filename, destination_path))
        tf.gfile.Copy(filename, destination_path)

Response: Take the latest checkpoint and copy it to model_path.


Instruction:
Write a Python 3 function for
initializing the weights from the given save_file.

Response:
def initialize_weights(self, save_file):
        """Initialize the weights from the given save_file.
        Assumes that the graph has been constructed, and the
        save_file contains weights that match the graph. Used
        to set the weights to a different version of the player
        without redifining the entire graph."""
        tf.train.Saver().restore(self.sess, save_file)


Instruction:
Can you create a Python 3 function that
parses the output of helpfull.

Response:
def parse_helpfull_output(help_output, regex=FLAG_HELP_RE_PY):
    """Parses the output of --helpfull.

    Args:
        help_output: str, the full output of --helpfull.

    Returns:
        A set of flags that are valid flags.
    """
    valid_flags = set()
    for _, no_prefix, flag_name in regex.findall(help_output):
        valid_flags.add('--' + flag_name)
        if no_prefix:
            valid_flags.add('--no' + flag_name)
    return valid_flags


Instruction:
Can you generate a brief explanation for the following Python 3 code
def filter_flags(parsed_flags, valid_flags):
    def valid_argv(argv):
        """Figures out if a flag parsed from the flagfile matches a flag in
        the command about to be run."""
        flagname_match = FLAG_RE.match(argv)
        if not flagname_match:
            return True
        flagname = flagname_match.group()
        return flagname in valid_flags
    return list(filter(valid_argv, parsed_flags))

Response: Return a subset of parsed_flags that are found in the list valid_flags.


Instruction:
Write a Python 3 function that can
prepare a subprocess command by running helpfull and masking flags.

Response:
def prepare_subprocess_cmd(subprocess_cmd):
    """Prepares a subprocess command by running --helpfull and masking flags.

    Args:
        subprocess_cmd: List[str], what would be passed into subprocess.call()
            i.e. ['python', 'train.py', '--flagfile=flags']

    Returns:
        ['python', 'train.py', '--train_flag=blah', '--more_flags']
    """
    help_cmd = subprocess_cmd + ['--helpfull']
    help_output = subprocess.run(help_cmd, stdout=subprocess.PIPE).stdout
    help_output = help_output.decode('ascii')
    if 'python' in subprocess_cmd[0]:
        valid_flags = parse_helpfull_output(help_output)
    else:
        valid_flags = parse_helpfull_output(help_output, regex=FLAG_HELP_RE_CC)
    parsed_flags = flags.FlagValues().read_flags_from_files(subprocess_cmd[1:])

    filtered_flags = filter_flags(parsed_flags, valid_flags)
    return [subprocess_cmd[0]] + filtered_flags


Instruction:
Can you implement a function in Python 3 that
prepares and run a subprocess cmd returning a CompletedProcess.

Response:
def run(cmd):
    """Prepare and run a subprocess cmd, returning a CompletedProcess."""
    print("Preparing the following cmd:")
    cmd = prepare_subprocess_cmd(cmd)
    print("Running the following cmd:")
    print('\n'.join(cmd))
    return subprocess.run(cmd, stdout=sys.stdout, stderr=sys.stderr)


Instruction:
Can you tell what is the following Python 3 function doing
def checked_run(cmd):
    completed_process = run(cmd)
    if completed_process.returncode > 0:
        print("Command failed!  Hanging around in case someone needs a "
              "docker connection. (Ctrl-C to quit now)")
        time.sleep(300)
        raise RuntimeError
    return completed_process

Response: Prepare and run a subprocess cmd checking for successful completion.


Instruction:
Given the following Python 3 function, write the documentation
def process_record_dataset(dataset, is_training, batch_size, shuffle_buffer,
                           parse_record_fn, num_epochs=1, num_gpus=None,
                           examples_per_epoch=None, dtype=tf.float32):
  """Given a Dataset with raw records, return an iterator over the records.

  Args:
    dataset: A Dataset representing raw records
    is_training: A boolean denoting whether the input is for training.
    batch_size: The number of samples per batch.
    shuffle_buffer: The buffer size to use when shuffling records. A larger
      value results in better randomness, but smaller values reduce startup
      time and use less memory.
    parse_record_fn: A function that takes a raw record and returns the
      corresponding (image, label) pair.
    num_epochs: The number of epochs to repeat the dataset.
    num_gpus: The number of gpus used for training.
    examples_per_epoch: The number of examples in an epoch.
    dtype: Data type to use for images/features.

  Returns:
    Dataset of (image, label) pairs ready for iteration.
  """

  # We prefetch a batch at a time, This can help smooth out the time taken to
  # load input files as we go through shuffling and processing.
  dataset = dataset.prefetch(buffer_size=batch_size)
  if is_training:
    # Shuffle the records. Note that we shuffle before repeating to ensure
    # that the shuffling respects epoch boundaries.
    mlperf_log.resnet_print(key=mlperf_log.INPUT_ORDER)
    dataset = dataset.shuffle(buffer_size=shuffle_buffer)

  # If we are training over multiple epochs before evaluating, repeat the
  # dataset for the appropriate number of epochs.
  dataset = dataset.repeat(num_epochs)

  # Parse the raw records into images and labels. Testing has shown that setting
  # num_parallel_batches > 1 produces no improvement in throughput, since
  # batch_size is almost always much greater than the number of CPU cores.
  dataset = dataset.apply(
      tf.contrib.data.map_and_batch(
          lambda value: parse_record_fn(value, is_training, dtype),
          batch_size=batch_size,
          num_parallel_batches=1))

  # Operations between the final prefetch and the get_next call to the iterator
  # will happen synchronously during run time. We prefetch here again to
  # background all of the above processing work and keep it out of the
  # critical training path. Setting buffer_size to tf.contrib.data.AUTOTUNE
  # allows DistributionStrategies to adjust how many batches to fetch based
  # on how many devices are present.
  dataset = dataset.prefetch(buffer_size=tf.contrib.data.AUTOTUNE)

  return dataset

Response: This function takes a Dataset representing raw records and returns an iterator over the records.