Instruction:
Make a summary of the following Python 3 code
def filter_data(self, min_len, max_len):
        logging.info(f'Filtering data, min len: {min_len}, max len: {max_len}')

        initial_len = len(self.src)
        filtered_src = []
        filtered_tgt = []
        for src, tgt in zip(self.src, self.tgt):
            if min_len <= len(src) <= max_len and \
                    min_len <= len(tgt) <= max_len:
                filtered_src.append(src)
                filtered_tgt.append(tgt)

        self.src = filtered_src
        self.tgt = filtered_tgt
        filtered_len = len(self.src)
        logging.info(f'Pairs before: {initial_len}, after: {filtered_len}')

Response: Filter the data for the inequality of the source and target sequences.


Instruction:
Can you generate the documentation for the following Python 3 function
def process_raw_data(self, fname, max_size):
        logging.info(f'Processing data from {fname}')
        data = []
        with open(fname) as dfile:
            for idx, line in enumerate(dfile):
                if max_size and idx == max_size:
                    break
                data.append(line)
        return data

Response: Loads the raw data from the input file fname.


Instruction:
Create a Python 3 function for
filtering out the raw data for the inequality of the source and target inequality.

Response:
def filter_raw_data(self, min_len, max_len):
        """
        Preserves only samples which satisfy the following inequality:
            min_len <= src sample sequence length <= max_len AND
            min_len <= tgt sample sequence length <= max_len

        :param min_len: minimum sequence length
        :param max_len: maximum sequence length
        """
        initial_len = len(self.raw_src)
        filtered_src = []
        filtered_tgt = []
        filtered_src_len = []
        filtered_tgt_len = []
        for src, tgt in zip(self.raw_src, self.raw_tgt):
            src_len = src.count(' ') + 1
            tgt_len = tgt.count(' ') + 1
            if min_len <= src_len <= max_len and \
                    min_len <= tgt_len <= max_len:
                filtered_src.append(src)
                filtered_tgt.append(tgt)
                filtered_src_len.append(src_len)
                filtered_tgt_len.append(tgt_len)

        self.raw_src = filtered_src
        self.raw_tgt = filtered_tgt
        self.src_len = filtered_src_len
        self.tgt_len = filtered_tgt_len
        filtered_len = len(self.raw_src)
        logging.info(f'Pairs before: {initial_len}, after: {filtered_len}')


Instruction:
Make a summary of the following Python 3 code
def heatmaps_to_keypoints(maps, rois):
    # This function converts a discrete image coordinate in a HEATMAP_SIZE x
    # HEATMAP_SIZE image to a continuous keypoint coordinate. We maintain
    # consistency with keypoints_to_heatmap_labels by using the conversion from
    # Heckbert 1990: c = d + 0.5, where d is a discrete coordinate and c is a
    # continuous coordinate.
    offset_x = rois[:, 0]
    offset_y = rois[:, 1]

    widths = rois[:, 2] - rois[:, 0]
    heights = rois[:, 3] - rois[:, 1]
    widths = np.maximum(widths, 1)
    heights = np.maximum(heights, 1)
    widths_ceil = np.ceil(widths)
    heights_ceil = np.ceil(heights)

    # NCHW to NHWC for use with OpenCV
    maps = np.transpose(maps, [0, 2, 3, 1])
    min_size = 0  # cfg.KRCNN.INFERENCE_MIN_SIZE
    num_keypoints = maps.shape[3]
    xy_preds = np.zeros((len(rois), 3, num_keypoints), dtype=np.float32)
    end_scores = np.zeros((len(rois), num_keypoints), dtype=np.float32)
    for i in range(len(rois)):
        if min_size > 0:
            roi_map_width = int(np.maximum(widths_ceil[i], min_size))
            roi_map_height = int(np.maximum(heights_ceil[i], min_size))
        else:
            roi_map_width = widths_ceil[i]
            roi_map_height = heights_ceil[i]
        width_correction = widths[i] / roi_map_width
        height_correction = heights[i] / roi_map_height
        roi_map = cv2.resize(
            maps[i], (roi_map_width, roi_map_height), interpolation=cv2.INTER_CUBIC
        )
        # Bring back to CHW
        roi_map = np.transpose(roi_map, [2, 0, 1])
        # roi_map_probs = scores_to_probs(roi_map.copy())
        w = roi_map.shape[2]
        pos = roi_map.reshape(num_keypoints, -1).argmax(axis=1)
        x_int = pos % w
        y_int = (pos - x_int) // w
        # assert (roi_map_probs[k, y_int, x_int] ==
        #         roi_map_probs[k, :, :].max())
        x = (x_int + 0.5) * width_correction
        y = (y_int + 0.5) * height_correction
        xy_preds[i, 0, :] = x + offset_x[i]
        xy_preds[i, 1, :] = y + offset_y[i]
        xy_preds[i, 2, :] = 1
        end_scores[i, :] = roi_map[np.arange(num_keypoints), y_int, x_int]

    return np.transpose(xy_preds, [0, 2, 1]), end_scores

Response: Extract predicted keypoint locations from heatmaps. Output has shape ( x y logit prob ) where x y is the keypoint location of each keypoint.


Instruction:
Can you implement a function in Python 3 that
builds a function that can be used during serving.

Response:
def build_tensor_serving_input_receiver_fn(shape, dtype=tf.float32,
                                           batch_size=1):
  """Returns a input_receiver_fn that can be used during serving.

  This expects examples to come through as float tensors, and simply
  wraps them as TensorServingInputReceivers.

  Arguably, this should live in tf.estimator.export. Testing here first.

  Args:
    shape: list representing target size of a single example.
    dtype: the expected datatype for the input example
    batch_size: number of input tensors that will be passed for prediction

  Returns:
    A function that itself returns a TensorServingInputReceiver.
  """
  def serving_input_receiver_fn():
    # Prep a placeholder where the input example will be fed in
    features = tf.placeholder(
        dtype=dtype, shape=[batch_size] + shape, name='input_tensor')

    return tf.estimator.export.TensorServingInputReceiver(
        features=features, receiver_tensors=features)

  return serving_input_receiver_fn


Instruction:
Can you create a Python 3 function that
validates which keypoints are contained inside a given box.

Response:
def _within_box(points, boxes):
    """Validate which keypoints are contained inside a given box.
    points: NxKx2
    boxes: Nx4
    output: NxK
    """
    x_within = (points[..., 0] >= boxes[:, 0, None]) & (
        points[..., 0] <= boxes[:, 2, None]
    )
    y_within = (points[..., 1] >= boxes[:, 1, None]) & (
        points[..., 1] <= boxes[:, 3, None]
    )
    return x_within & y_within


Instruction:
Make a summary of the following Python 3 code
def begin(self):
    self._global_step_tensor = tf.train.get_global_step()
    if self._global_step_tensor is None:
      raise RuntimeError(
          'Global step should be created to use StepCounterHook.')

Response: Called once before using the session to check global step.


Instruction:
Can you tell what is the following Python 3 function doing
def after_run(self, run_context, run_values):  # pylint: disable=unused-argument
    global_step = run_values.results

    if self._timer.should_trigger_for_step(
        global_step) and global_step > self._warm_steps:
      elapsed_time, elapsed_steps = self._timer.update_last_triggered_step(
          global_step)
      if elapsed_time is not None:
        self._step_train_time += elapsed_time
        self._total_steps += elapsed_steps

        # average examples per second is based on the total (accumulative)
        # training steps and training time so far
        average_examples_per_sec = self._batch_size * (
            self._total_steps / self._step_train_time)
        # current examples per second is based on the elapsed training steps
        # and training time per batch
        current_examples_per_sec = self._batch_size * (
            elapsed_steps / elapsed_time)
        # Current examples/sec followed by average examples/sec
        tf.logging.info('Batch [%g]:  current exp/sec = %g, average exp/sec = '
                        '%g', self._total_steps, current_examples_per_sec,
                        average_examples_per_sec)

Response: Called after each run.


Instruction:
Implement a Python 3 function for
visualizing keypoints in a single image.

Response:
def vis_keypoints(img, kps, kp_thresh=2, alpha=0.7):
    """Visualizes keypoints (adapted from vis_one_image).
    kps has shape (4, #keypoints) where 4 rows are (x, y, logit, prob).
    """
    dataset_keypoints = PersonKeypoints.NAMES
    kp_lines = PersonKeypoints.CONNECTIONS

    # Convert from plt 0-1 RGBA colors to 0-255 BGR colors for opencv.
    cmap = plt.get_cmap('rainbow')
    colors = [cmap(i) for i in np.linspace(0, 1, len(kp_lines) + 2)]
    colors = [(c[2] * 255, c[1] * 255, c[0] * 255) for c in colors]

    # Perform the drawing on a copy of the image, to allow for blending.
    kp_mask = np.copy(img)

    # Draw mid shoulder / mid hip first for better visualization.
    mid_shoulder = (
        kps[:2, dataset_keypoints.index('right_shoulder')] +
        kps[:2, dataset_keypoints.index('left_shoulder')]) / 2.0
    sc_mid_shoulder = np.minimum(
        kps[2, dataset_keypoints.index('right_shoulder')],
        kps[2, dataset_keypoints.index('left_shoulder')])
    mid_hip = (
        kps[:2, dataset_keypoints.index('right_hip')] +
        kps[:2, dataset_keypoints.index('left_hip')]) / 2.0
    sc_mid_hip = np.minimum(
        kps[2, dataset_keypoints.index('right_hip')],
        kps[2, dataset_keypoints.index('left_hip')])
    nose_idx = dataset_keypoints.index('nose')
    if sc_mid_shoulder > kp_thresh and kps[2, nose_idx] > kp_thresh:
        cv2.line(
            kp_mask, tuple(mid_shoulder), tuple(kps[:2, nose_idx]),
            color=colors[len(kp_lines)], thickness=2, lineType=cv2.LINE_AA)
    if sc_mid_shoulder > kp_thresh and sc_mid_hip > kp_thresh:
        cv2.line(
            kp_mask, tuple(mid_shoulder), tuple(mid_hip),
            color=colors[len(kp_lines) + 1], thickness=2, lineType=cv2.LINE_AA)

    # Draw the keypoints.
    for l in range(len(kp_lines)):
        i1 = kp_lines[l][0]
        i2 = kp_lines[l][1]
        p1 = kps[0, i1], kps[1, i1]
        p2 = kps[0, i2], kps[1, i2]
        if kps[2, i1] > kp_thresh and kps[2, i2] > kp_thresh:
            cv2.line(
                kp_mask, p1, p2,
                color=colors[l], thickness=2, lineType=cv2.LINE_AA)
        if kps[2, i1] > kp_thresh:
            cv2.circle(
                kp_mask, p1,
                radius=3, color=colors[l], thickness=-1, lineType=cv2.LINE_AA)
        if kps[2, i2] > kp_thresh:
            cv2.circle(
                kp_mask, p2,
                radius=3, color=colors[l], thickness=-1, lineType=cv2.LINE_AA)

    # Blend the keypoints.
    return cv2.addWeighted(img, 1.0 - alpha, kp_mask, alpha, 0)


Instruction:
Can you generate the documentation for the following Python 3 function
def build_transform(self):
        cfg = self.cfg

        # we are loading images with OpenCV, so we don't need to convert them
        # to BGR, they are already! So all we need to do is to normalize
        # by 255 if we want to convert to BGR255 format, or flip the channels
        # if we want it to be in RGB in [0-1] range.
        if cfg.INPUT.TO_BGR255:
            to_bgr_transform = T.Lambda(lambda x: x * 255)
        else:
            to_bgr_transform = T.Lambda(lambda x: x[[2, 1, 0]])

        normalize_transform = T.Normalize(
            mean=cfg.INPUT.PIXEL_MEAN, std=cfg.INPUT.PIXEL_STD
        )

        transform = T.Compose(
            [
                T.ToPILImage(),
                T.Resize(self.min_image_size),
                T.ToTensor(),
                to_bgr_transform,
                normalize_transform,
            ]
        )
        return transform

Response: Builds a basic transformation that was used to train the models.


Instruction:
Explain what the following Python 3 code does
def run_on_opencv_image(self, image):
        predictions = self.compute_prediction(image)
        top_predictions = self.select_top_predictions(predictions)

        result = image.copy()
        if self.show_mask_heatmaps:
            return self.create_mask_montage(result, top_predictions)
        result = self.overlay_boxes(result, top_predictions)
        if self.cfg.MODEL.MASK_ON:
            result = self.overlay_mask(result, top_predictions)
        if self.cfg.MODEL.KEYPOINT_ON:
            result = self.overlay_keypoints(result, top_predictions)
        result = self.overlay_class_names(result, top_predictions)

        return result

Response: Runs the detection on an image.


Instruction:
Can you implement a function in Python 3 that
computes the prediction of a single image.

Response:
def compute_prediction(self, original_image):
        """
        Arguments:
            original_image (np.ndarray): an image as returned by OpenCV

        Returns:
            prediction (BoxList): the detected objects. Additional information
                of the detection properties can be found in the fields of
                the BoxList via `prediction.fields()`
        """
        # apply pre-processing to image
        image = self.transforms(original_image)
        # convert to an ImageList, padded so that it is divisible by
        # cfg.DATALOADER.SIZE_DIVISIBILITY
        image_list = to_image_list(image, self.cfg.DATALOADER.SIZE_DIVISIBILITY)
        image_list = image_list.to(self.device)
        # compute predictions
        with torch.no_grad():
            predictions = self.model(image_list)
        predictions = [o.to(self.cpu_device) for o in predictions]

        # always single image is passed at a time
        prediction = predictions[0]

        # reshape prediction (a BoxList) into the original image size
        height, width = original_image.shape[:-1]
        prediction = prediction.resize((width, height))

        if prediction.has_field("mask"):
            # if we have masks, paste the masks in the right position
            # in the image, as defined by the bounding boxes
            masks = prediction.get_field("mask")
            # always single image is passed at a time
            masks = self.masker([masks], [prediction])[0]
            prediction.add_field("mask", masks)
        return prediction


Instruction:
Make a summary of the following Python 3 code
def select_top_predictions(self, predictions):
        scores = predictions.get_field("scores")
        keep = torch.nonzero(scores > self.confidence_threshold).squeeze(1)
        predictions = predictions[keep]
        scores = predictions.get_field("scores")
        _, idx = scores.sort(0, descending=True)
        return predictions[idx]

Response: Select only predictions which have a score > self. confidence_threshold and returns the predictions in descending order of score


Instruction:
Make a summary of the following Python 3 code
def compute_colors_for_labels(self, labels):
        colors = labels[:, None] * self.palette
        colors = (colors % 255).numpy().astype("uint8")
        return colors

Response: Simple function that adds fixed colors depending on the class


Instruction:
Can you implement a function in Python 3 that
overlays the boxes of the image with the predicted boxes.

Response:
def overlay_boxes(self, image, predictions):
        """
        Adds the predicted boxes on top of the image

        Arguments:
            image (np.ndarray): an image as returned by OpenCV
            predictions (BoxList): the result of the computation by the model.
                It should contain the field `labels`.
        """
        labels = predictions.get_field("labels")
        boxes = predictions.bbox

        colors = self.compute_colors_for_labels(labels).tolist()

        for box, color in zip(boxes, colors):
            box = box.to(torch.int64)
            top_left, bottom_right = box[:2].tolist(), box[2:].tolist()
            image = cv2.rectangle(
                image, tuple(top_left), tuple(bottom_right), tuple(color), 1
            )

        return image


Instruction:
Create a Python 3 function for
overlaying the image with the mask and labels.

Response:
def overlay_mask(self, image, predictions):
        """
        Adds the instances contours for each predicted object.
        Each label has a different color.

        Arguments:
            image (np.ndarray): an image as returned by OpenCV
            predictions (BoxList): the result of the computation by the model.
                It should contain the field `mask` and `labels`.
        """
        masks = predictions.get_field("mask").numpy()
        labels = predictions.get_field("labels")

        colors = self.compute_colors_for_labels(labels).tolist()

        for mask, color in zip(masks, colors):
            thresh = mask[0, :, :, None]
            contours, hierarchy = cv2_util.findContours(
                thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE
            )
            image = cv2.drawContours(image, contours, -1, color, 3)

        composite = image

        return composite


Instruction:
Can you generate the documentation for the following Python 3 function
def create_mask_montage(self, image, predictions):
        masks = predictions.get_field("mask")
        masks_per_dim = self.masks_per_dim
        masks = L.interpolate(
            masks.float(), scale_factor=1 / masks_per_dim
        ).byte()
        height, width = masks.shape[-2:]
        max_masks = masks_per_dim ** 2
        masks = masks[:max_masks]
        # handle case where we have less detections than max_masks
        if len(masks) < max_masks:
            masks_padded = torch.zeros(max_masks, 1, height, width, dtype=torch.uint8)
            masks_padded[: len(masks)] = masks
            masks = masks_padded
        masks = masks.reshape(masks_per_dim, masks_per_dim, height, width)
        result = torch.zeros(
            (masks_per_dim * height, masks_per_dim * width), dtype=torch.uint8
        )
        for y in range(masks_per_dim):
            start_y = y * height
            end_y = (y + 1) * height
            for x in range(masks_per_dim):
                start_x = x * width
                end_x = (x + 1) * width
                result[start_y:end_y, start_x:end_x] = masks[y, x]
        return cv2.applyColorMap(result.numpy(), cv2.COLORMAP_JET)

Response: Create a montage showing the probability heatmaps for each one of the objects in the image.


Instruction:
Given the following Python 3 function, write the documentation
def overlay_class_names(self, image, predictions):
        scores = predictions.get_field("scores").tolist()
        labels = predictions.get_field("labels").tolist()
        labels = [self.CATEGORIES[i] for i in labels]
        boxes = predictions.bbox

        template = "{}: {:.2f}"
        for box, score, label in zip(boxes, scores, labels):
            x, y = box[:2]
            s = template.format(label, score)
            cv2.putText(
                image, s, (x, y), cv2.FONT_HERSHEY_SIMPLEX, .5, (255, 255, 255), 1
            )

        return image

Response: Overlays detected class names and scores in the positions defined by the top - left corner of the predicted bounding box.


Instruction:
Can you write a function in Python 3 where it
collects the CPU information for the local environment.

Response:
def _collect_cpu_info(run_info):
  """Collect the CPU information for the local environment."""
  cpu_info = {}

  cpu_info["num_cores"] = multiprocessing.cpu_count()

  # Note: cpuinfo is not installed in the TensorFlow OSS tree.
  # It is installable via pip.
  import cpuinfo    # pylint: disable=g-import-not-at-top

  info = cpuinfo.get_cpu_info()
  cpu_info["cpu_info"] = info["brand"]
  cpu_info["mhz_per_cpu"] = info["hz_advertised_raw"][0] / 1.0e6

  run_info["machine_config"]["cpu_info"] = cpu_info


Instruction:
How would you implement a function in Python 3 that
collects GPU information by TF device library.

Response:
def _collect_gpu_info(run_info):
  """Collect local GPU information by TF device library."""
  gpu_info = {}
  local_device_protos = device_lib.list_local_devices()

  gpu_info["count"] = len([d for d in local_device_protos
                           if d.device_type == "GPU"])
  # The device description usually is a JSON string, which contains the GPU
  # model info, eg:
  # "device: 0, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:04.0"
  for d in local_device_protos:
    if d.device_type == "GPU":
      gpu_info["model"] = _parse_gpu_model(d.physical_device_desc)
      # Assume all the GPU connected are same model
      break
  run_info["machine_config"]["gpu_info"] = gpu_info


Instruction:
How would you explain what the following Python 3 function does
def log_estimator_evaluation_result(self, eval_results):
    if not isinstance(eval_results, dict):
      tf.logging.warning("eval_results should be directory for logging. Got %s",
                         type(eval_results))
      return
    global_step = eval_results[tf.GraphKeys.GLOBAL_STEP]
    for key in sorted(eval_results):
      if key != tf.GraphKeys.GLOBAL_STEP:
        self.log_metric(key, eval_results[key], global_step=global_step)

Response: Log the evaluation result for a estimator.


Instruction:
Explain what the following Python 3 code does
def log_metric(self, name, value, unit=None, global_step=None, extras=None):
    if not isinstance(value, numbers.Number):
      tf.logging.warning(
          "Metric value to log should be a number. Got %s", type(value))
      return
    if extras:
      extras = [{"name": k, "value": v} for k, v in sorted(extras.items())]
    else:
      extras = []
    with tf.gfile.GFile(
        os.path.join(self._logging_dir, METRIC_LOG_FILE_NAME), "a") as f:
      metric = {
          "name": name,
          "value": float(value),
          "unit": unit,
          "global_step": global_step,
          "timestamp": datetime.datetime.now().strftime(
              _DATE_TIME_FORMAT_PATTERN),
          "extras": extras}
      try:
        json.dump(metric, f)
        f.write("\n")
      except (TypeError, ValueError) as e:
        tf.logging.warning("Failed to dump metric to log file: "
                           "name %s, value %s, error %s", name, value, e)

Response: Logs the metric to the local file.


Instruction:
Write a Python 3 script to
log the run info for the local env.

Response:
def log_run_info(self, model_name):
    """Collect most of the TF runtime information for the local env.

    The schema of the run info follows official/benchmark/datastore/schema.

    Args:
      model_name: string, the name of the model.
    """
    run_info = {
        "model_name": model_name,
        "machine_config": {},
        "run_date": datetime.datetime.now().strftime(_DATE_TIME_FORMAT_PATTERN)}
    _collect_tensorflow_info(run_info)
    _collect_tensorflow_environment_variables(run_info)
    _collect_cpu_info(run_info)
    _collect_gpu_info(run_info)
    _collect_memory_info(run_info)

    with tf.gfile.GFile(os.path.join(
        self._logging_dir, BENCHMARK_RUN_LOG_FILE_NAME), "w") as f:
      try:
        json.dump(run_info, f)
        f.write("\n")
      except (TypeError, ValueError) as e:
        tf.logging.warning("Failed to dump benchmark run info to log file: %s",
                           e)


Instruction:
Here you have a function in Python 3, explain what it does
def read_existing_paths(bt_table):
    rows = bt_table.read_rows(
        filter_=row_filters.ColumnRangeFilter(
            METADATA, SGF_FILENAME, SGF_FILENAME))
    names = (row.cell_value(METADATA, SGF_FILENAME).decode() for row in rows)
    processed = [os.path.splitext(os.path.basename(r))[0] for r in names]
    return processed

Response: Return the SGF filename for each existing eval record.


Instruction:
Can you generate the documentation for the following Python 3 function
def canonical_name(sgf_name):
    sgf_name = os.path.normpath(sgf_name)
    assert sgf_name.endswith('.sgf'), sgf_name
    # Strip off '.sgf'
    sgf_name = sgf_name[:-4]

    # Often eval is inside a folder with the run name.
    # include from folder before /eval/ if part of path.
    with_folder = re.search(r'/([^/]*/eval/.*)', sgf_name)
    if with_folder:
        return with_folder.group(1)

    # Return the filename
    return os.path.basename(sgf_name)

Response: Keep filename and some date folders


Instruction:
Can you create a Python 3 function that
reads all SGFs that match glob and extract relevant metadata for eval games table.

Response:
def read_games(glob, existing_paths):
    """Read all SGFs that match glob

    Parse each game and extract relevant metadata for eval games table.
    """

    globbed = sorted(gfile.Glob(glob))

    skipped = 0
    to_parse = []
    for sgf_name in tqdm(globbed):
        assert sgf_name.lower().endswith('.sgf'), sgf_name
        sgf_path = canonical_name(sgf_name)
        sgf_filename = os.path.basename(sgf_path)

        if sgf_path in existing_paths or sgf_filename in existing_paths:
            skipped += 1
            continue

        to_parse.append(sgf_name)

    game_data = []
    with multiprocessing.Pool() as pool:
        game_data = pool.map(process_game, tqdm(to_parse), 100)

    print("Read {} SGFs, {} new, {} existing".format(
        len(globbed), len(game_data), skipped))
    return game_data


Instruction:
Given the following Python 3 function, write the documentation
def write_eval_records(bt_table, game_data, last_game):
    eval_num = last_game

    # Each column counts as a mutation so max rows is ~10000
    GAMES_PER_COMMIT = 2000
    for games in grouper(tqdm(game_data), GAMES_PER_COMMIT):
        assert bt_table.read_row(EVAL_PREFIX.format(eval_num)), "Prev row doesn't exists"
        assert bt_table.read_row(EVAL_PREFIX.format(eval_num+1)) is None, "Row already exists"

        rows = []
        for i, metadata in enumerate(games):
            eval_num += 1
            row_name = EVAL_PREFIX.format(eval_num)
            row = bt_table.row(row_name)
            for column, value in metadata:
                row.set_cell(METADATA, column, value)
            rows.append(row)
            # For each batch of games print a couple of the rows being added.
            if i < 5 or i + 5 > len(games):
                print("\t", i, row_name, metadata[6][1])

        if eval_num == last_game + len(games):
            test = input("Commit ('y'/'yes' required): ")
            if test.lower() not in ('y', 'yes'):
                break

        # TODO(derek): Figure out how to condition on atomic counter update.
        # Condition all updates on the current value of last_game

        game_num_update = bt_table.row(TABLE_STATE)
        game_num_update.set_cell(METADATA, EVAL_GAME_COUNTER, eval_num)
        print(TABLE_STATE, eval_num)

        response = bt_table.mutate_rows(rows)

        # validate that all rows written successfully
        any_bad = False
        for i, status in enumerate(response):
            if status.code is not 0:
                print("Row number {} failed to write {}".format(i, status))
                any_bad = True
        if any_bad:
            break

        game_num_update.commit()

Response: Write all eval records to eval_table.


Instruction:
Implement a function in Python 3 to
initialize the reinforcement learning loop from a checkpoint.

Response:
def initialize_from_checkpoint(state):
  """Initialize the reinforcement learning loop from a checkpoint."""

  # The checkpoint's work_dir should contain the most recently trained model.
  model_paths = glob.glob(os.path.join(FLAGS.checkpoint_dir,
                                       'work_dir/model.ckpt-*.pb'))
  if len(model_paths) != 1:
    raise RuntimeError('Expected exactly one model in the checkpoint work_dir, '
                       'got [{}]'.format(', '.join(model_paths)))
  start_model_path = model_paths[0]

  # Copy the latest trained model into the models directory and use it on the
  # first round of selfplay.
  state.best_model_name = 'checkpoint'
  shutil.copy(start_model_path,
              os.path.join(fsdb.models_dir(), state.best_model_name + '.pb'))

  # Copy the training chunks.
  golden_chunks_dir = os.path.join(FLAGS.checkpoint_dir, 'golden_chunks')
  for basename in os.listdir(golden_chunks_dir):
    path = os.path.join(golden_chunks_dir, basename)
    shutil.copy(path, fsdb.golden_chunk_dir())

  # Copy the training files.
  work_dir = os.path.join(FLAGS.checkpoint_dir, 'work_dir')
  for basename in os.listdir(work_dir):
    path = os.path.join(work_dir, basename)
    shutil.copy(path, fsdb.working_dir())


Instruction:
Write a Python 3 function that can
run the given subprocess command in a coroutine.

Response:
async def run(*cmd):
  """Run the given subprocess command in a coroutine.

  Args:
    *cmd: the command to run and its arguments.

  Returns:
    The output that the command wrote to stdout as a list of strings, one line
    per element (stderr output is piped to stdout).

  Raises:
    RuntimeError: if the command returns a non-zero result.
  """

  stdout = await checked_run(*cmd)

  log_path = os.path.join(FLAGS.base_dir, get_cmd_name(cmd) + '.log')
  with gfile.Open(log_path, 'a') as f:
    f.write(expand_cmd_str(cmd))
    f.write('\n')
    f.write(stdout)
    f.write('\n')

  # Split stdout into lines.
  return stdout.split('\n')


Instruction:
Write a Python 3 script for
returning up to num_records of golden chunks to train on.

Response:
def get_golden_chunk_records():
  """Return up to num_records of golden chunks to train on.

  Returns:
    A list of golden chunks up to num_records in length, sorted by path.
  """

  pattern = os.path.join(fsdb.golden_chunk_dir(), '*.zz')
  return sorted(tf.gfile.Glob(pattern), reverse=True)[:FLAGS.window_size]


Instruction:
How would you explain what the following Python 3 function does
async def selfplay(state, flagfile='selfplay'):

  output_dir = os.path.join(fsdb.selfplay_dir(), state.output_model_name)
  holdout_dir = os.path.join(fsdb.holdout_dir(), state.output_model_name)

  lines = await run(
      'bazel-bin/cc/selfplay',
      '--flagfile={}.flags'.format(os.path.join(FLAGS.flags_dir, flagfile)),
      '--model={}'.format(state.best_model_path),
      '--output_dir={}'.format(output_dir),
      '--holdout_dir={}'.format(holdout_dir),
      '--seed={}'.format(state.seed))
  result = '\n'.join(lines[-6:])
  logging.info(result)
  stats = parse_win_stats_table(result, 1)[0]
  num_games = stats.total_wins
  logging.info('Black won %0.3f, white won %0.3f',
               stats.black_wins.total / num_games,
               stats.white_wins.total / num_games)

  # Write examples to a single record.
  pattern = os.path.join(output_dir, '*', '*.zz')
  random.seed(state.seed)
  tf.set_random_seed(state.seed)
  np.random.seed(state.seed)
  # TODO(tommadams): This method of generating one golden chunk per generation
  # is sub-optimal because each chunk gets reused multiple times for training,
  # introducing bias. Instead, a fresh dataset should be uniformly sampled out
  # of *all* games in the training window before the start of each training run.
  buffer = example_buffer.ExampleBuffer(sampling_frac=1.0)

  # TODO(tommadams): parallel_fill is currently non-deterministic. Make it not
  # so.
  logging.info('Writing golden chunk from "{}"'.format(pattern))
  buffer.parallel_fill(tf.gfile.Glob(pattern))
  buffer.flush(os.path.join(fsdb.golden_chunk_dir(),
                            state.output_model_name + '.tfrecord.zz'))

Response: Run selfplay and write a training chunk to the fsdb golden_chunk_dir.


Instruction:
Write a Python 3 function that can
train a new model on the RL loop.

Response:
async def train(state, tf_records):
  """Run training and write a new model to the fsdb models_dir.

  Args:
    state: the RL loop State instance.
    tf_records: a list of paths to TensorFlow records to train on.
  """

  model_path = os.path.join(fsdb.models_dir(), state.train_model_name)
  await run(
      'python3', 'train.py', *tf_records,
      '--flagfile={}'.format(os.path.join(FLAGS.flags_dir, 'train.flags')),
      '--work_dir={}'.format(fsdb.working_dir()),
      '--export_path={}'.format(model_path),
      '--training_seed={}'.format(state.seed),
      '--freeze=true')
  # Append the time elapsed from when the RL was started to when this model
  # was trained.
  elapsed = time.time() - state.start_time
  timestamps_path = os.path.join(fsdb.models_dir(), 'train_times.txt')
  with gfile.Open(timestamps_path, 'a') as f:
    print('{:.3f} {}'.format(elapsed, state.train_model_name), file=f)


Instruction:
Write a Python 3 script for
validating the trained model against holdout games.

Response:
async def validate(state, holdout_glob):
  """Validate the trained model against holdout games.

  Args:
    state: the RL loop State instance.
    holdout_glob: a glob that matches holdout games.
  """

  if not glob.glob(holdout_glob):
    print('Glob "{}" didn\'t match any files, skipping validation'.format(
          holdout_glob))
  else:
    await run(
        'python3', 'validate.py', holdout_glob,
        '--flagfile={}'.format(os.path.join(FLAGS.flags_dir, 'validate.flags')),
        '--work_dir={}'.format(fsdb.working_dir()))


Instruction:
Can you tell what is the following Python 3 function doing
async def evaluate_model(eval_model_path, target_model_path, sgf_dir, seed):

  lines = await run(
      'bazel-bin/cc/eval',
      '--flagfile={}'.format(os.path.join(FLAGS.flags_dir, 'eval.flags')),
      '--model={}'.format(eval_model_path),
      '--model_two={}'.format(target_model_path),
      '--sgf_dir={}'.format(sgf_dir),
      '--seed={}'.format(seed))
  result = '\n'.join(lines[-7:])
  logging.info(result)
  eval_stats, target_stats = parse_win_stats_table(result, 2)
  num_games = eval_stats.total_wins + target_stats.total_wins
  win_rate = eval_stats.total_wins / num_games
  logging.info('Win rate %s vs %s: %.3f', eval_stats.model_name,
               target_stats.model_name, win_rate)
  return win_rate

Response: Evaluate one model against a target.


Instruction:
How would you explain what the following Python 3 function does
async def evaluate_trained_model(state):

  return await evaluate_model(
      state.train_model_path, state.best_model_path,
      os.path.join(fsdb.eval_dir(), state.train_model_name), state.seed)

Response: Evaluate the most recently trained model against the current best model.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def rl_loop():

  state = State()

  if FLAGS.checkpoint_dir:
    # Start from a partially trained model.
    initialize_from_checkpoint(state)
  else:
    # Play the first round of selfplay games with a fake model that returns
    # random noise. We do this instead of playing multiple games using a single
    # model bootstrapped with random noise to avoid any initial bias.
    wait(selfplay(state, 'bootstrap'))

    # Train a real model from the random selfplay games.
    tf_records = get_golden_chunk_records()
    state.iter_num += 1
    wait(train(state, tf_records))

    # Select the newly trained model as the best.
    state.best_model_name = state.train_model_name
    state.gen_num += 1

    # Run selfplay using the new model.
    wait(selfplay(state))

  # Now start the full training loop.
  while state.iter_num <= FLAGS.iterations:
    # Build holdout glob before incrementing the iteration number because we
    # want to run validation on the previous generation.
    holdout_glob = os.path.join(fsdb.holdout_dir(), '%06d-*' % state.iter_num,
                                '*')

    # Train on shuffled game data from recent selfplay rounds.
    tf_records = get_golden_chunk_records()
    state.iter_num += 1
    wait(train(state, tf_records))

    if FLAGS.parallel_post_train:
      # Run eval, validation & selfplay in parallel.
      model_win_rate, _, _ = wait([
          evaluate_trained_model(state),
          validate(state, holdout_glob),
          selfplay(state)])
    else:
      # Run eval, validation & selfplay sequentially.
      model_win_rate = wait(evaluate_trained_model(state))
      wait(validate(state, holdout_glob))
      wait(selfplay(state))

    if model_win_rate >= FLAGS.gating_win_rate:
      # Promote the trained model to the best model and increment the generation
      # number.
      state.best_model_name = state.train_model_name
      state.gen_num += 1

Response: The main reinforcement learning loop.


Instruction:
Implement a function in Python 3 to
run the reinforcement learning loop.

Response:
def main(unused_argv):
  """Run the reinforcement learning loop."""

  print('Wiping dir %s' % FLAGS.base_dir, flush=True)
  shutil.rmtree(FLAGS.base_dir, ignore_errors=True)
  dirs = [fsdb.models_dir(), fsdb.selfplay_dir(), fsdb.holdout_dir(),
          fsdb.eval_dir(), fsdb.golden_chunk_dir(), fsdb.working_dir()]
  for d in dirs:
    ensure_dir_exists(d);

  # Copy the flag files so there's no chance of them getting accidentally
  # overwritten while the RL loop is running.
  flags_dir = os.path.join(FLAGS.base_dir, 'flags')
  shutil.copytree(FLAGS.flags_dir, flags_dir)
  FLAGS.flags_dir = flags_dir

  # Copy the target model to the models directory so we can find it easily.
  shutil.copy(FLAGS.target_path, os.path.join(fsdb.models_dir(), 'target.pb'))

  logging.getLogger().addHandler(
      logging.FileHandler(os.path.join(FLAGS.base_dir, 'rl_loop.log')))
  formatter = logging.Formatter('[%(asctime)s] %(message)s',
                                '%Y-%m-%d %H:%M:%S')
  for handler in logging.getLogger().handlers:
    handler.setFormatter(formatter)

  with logged_timer('Total time'):
    try:
      rl_loop()
    finally:
      asyncio.get_event_loop().close()


Instruction:
Implement a function in Python 3 to
yield a sequence of PositionWithContext objects for each player in the position.

Response:
def replay_position(position, result):
    """
    Wrapper for a go.Position which replays its history.
    Assumes an empty start position! (i.e. no handicap, and history must be exhaustive.)

    Result must be passed in, since a resign cannot be inferred from position
    history alone.

    for position_w_context in replay_position(position):
        print(position_w_context.position)
    """
    assert position.n == len(position.recent), "Position history is incomplete"
    pos = Position(komi=position.komi)
    for player_move in position.recent:
        color, next_move = player_move
        yield PositionWithContext(pos, next_move, result)
        pos = pos.play_move(next_move, color=color)


Instruction:
Can you create a Python 3 function that
checks if c is surrounded on all sides by 1 color and return that color

Response:
def is_koish(board, c):
    'Check if c is surrounded on all sides by 1 color, and return that color'
    if board[c] != EMPTY:
        return None
    neighbors = {board[n] for n in NEIGHBORS[c]}
    if len(neighbors) == 1 and EMPTY not in neighbors:
        return list(neighbors)[0]
    else:
        return None


Instruction:
Write a Python 3 function that can
check if c is an eye for the purpose of restricting MC rollouts.

Response:
def is_eyeish(board, c):
    'Check if c is an eye, for the purpose of restricting MC rollouts.'
    # pass is fine.
    if c is None:
        return
    color = is_koish(board, c)
    if color is None:
        return None
    diagonal_faults = 0
    diagonals = DIAGONALS[c]
    if len(diagonals) < 4:
        diagonal_faults += 1
    for d in diagonals:
        if not board[d] in (color, EMPTY):
            diagonal_faults += 1
    if diagonal_faults > 1:
        return None
    else:
        return color


Instruction:
Here you have a function in Python 3, explain what it does
def is_move_legal(self, move):
        'Checks that a move is on an empty space, not on ko, and not suicide'
        if move is None:
            return True
        if self.board[move] != EMPTY:
            return False
        if move == self.ko:
            return False
        if self.is_move_suicidal(move):
            return False

        return True

Response: Checks that a move is on an empty space not on ko and not suicide


Instruction:
Can you generate the documentation for the following Python 3 function
def all_legal_moves(self):
        'Returns a np.array of size go.N**2 + 1, with 1 = legal, 0 = illegal'
        # by default, every move is legal
        legal_moves = np.ones([N, N], dtype=np.int8)
        # ...unless there is already a stone there
        legal_moves[self.board != EMPTY] = 0
        # calculate which spots have 4 stones next to them
        # padding is because the edge always counts as a lost liberty.
        adjacent = np.ones([N + 2, N + 2], dtype=np.int8)
        adjacent[1:-1, 1:-1] = np.abs(self.board)
        num_adjacent_stones = (adjacent[:-2, 1:-1] + adjacent[1:-1, :-2] +
                               adjacent[2:, 1:-1] + adjacent[1:-1, 2:])
        # Surrounded spots are those that are empty and have 4 adjacent stones.
        surrounded_spots = np.multiply(
            (self.board == EMPTY),
            (num_adjacent_stones == 4))
        # Such spots are possibly illegal, unless they are capturing something.
        # Iterate over and manually check each spot.
        for coord in np.transpose(np.nonzero(surrounded_spots)):
            if self.is_move_suicidal(tuple(coord)):
                legal_moves[tuple(coord)] = 0

        # ...and retaking ko is always illegal
        if self.ko is not None:
            legal_moves[self.ko] = 0

        # and pass is always legal
        return np.concatenate([legal_moves.ravel(), [1]])

Response: Returns a np. array of size go. N ** 2 + 1 with 1 = legal 0 = illegal


Instruction:
Write a Python 3 function that can
return score from B perspective. If W is winning score is negative.

Response:
def score(self):
        'Return score from B perspective. If W is winning, score is negative.'
        working_board = np.copy(self.board)
        while EMPTY in working_board:
            unassigned_spaces = np.where(working_board == EMPTY)
            c = unassigned_spaces[0][0], unassigned_spaces[1][0]
            territory, borders = find_reached(working_board, c)
            border_colors = set(working_board[b] for b in borders)
            X_border = BLACK in border_colors
            O_border = WHITE in border_colors
            if X_border and not O_border:
                territory_color = BLACK
            elif O_border and not X_border:
                territory_color = WHITE
            else:
                territory_color = UNKNOWN  # dame, or seki
            place_stones(working_board, territory_color, territory)

        return np.count_nonzero(working_board == BLACK) - np.count_nonzero(working_board == WHITE) - self.komi


Instruction:
Explain what the following Python 3 code does
def parse_sgf_to_examples(sgf_path):

    return zip(*[(p.position, p.next_move, p.result)
                 for p in sgf_wrapper.replay_sgf_file(sgf_path)])

Response: Parse a SGF file and return a list of examples.


Instruction:
Write a Python 3 function that can
find all sgf files in base_dir with year >= min_year and komi

Response:
def find_and_filter_sgf_files(base_dir, min_year=None, komi=None):
    """Finds all sgf files in base_dir with year >= min_year and komi"""
    sgf_files = []
    for dirpath, dirnames, filenames in os.walk(base_dir):
        for filename in filenames:
            if filename.endswith('.sgf'):
                path = os.path.join(dirpath, filename)
                sgf_files.append(path)

    if min_year == komi == None:
        print ("Found {} sgf_files".format(len(sgf_files)))
        return sgf_files

    f = filter_year_komi(min_year, komi)
    filtered_sgf_files = [sgf for sgf in tqdm(sgf_files) if f(sgf)]

    print("{} of {} .sgf files matched (min_year >= {}, komi = {})".format(
        len(filtered_sgf_files), len(sgf_files), min_year, komi))
    return filtered_sgf_files


Instruction:
Can you tell what is the following Python 3 function doing
def get_model_paths(model_dir):
    all_models = gfile.Glob(os.path.join(model_dir, '*.meta'))
    model_filenames = [os.path.basename(m) for m in all_models]
    model_numbers_names = [
        (shipname.detect_model_num(m), shipname.detect_model_name(m))
        for m in model_filenames]
    model_names = sorted(model_numbers_names)
    return [os.path.join(model_dir, name[1]) for name in model_names]

Response: Returns all model paths in the model_dir.


Instruction:
Implement a Python 3 function for
turning a game into SGF.

Response:
def make_sgf(
    move_history,
    result_string,
    ruleset="Chinese",
    komi=7.5,
    white_name=PROGRAM_IDENTIFIER,
    black_name=PROGRAM_IDENTIFIER,
    comments=[]
):
    """Turn a game into SGF.

    Doesn't handle handicap games or positions with incomplete history.

    Args:
        move_history: iterable of PlayerMoves
        result_string: "B+R", "W+0.5", etc.
        comments: iterable of string/None. Will be zipped with move_history.
    """
    boardsize = go.N
    game_moves = ''.join(translate_sgf_move(*z)
                         for z in itertools.zip_longest(move_history, comments))
    result = result_string
    return SGF_TEMPLATE.format(**locals())


Instruction:
How would you explain what the following Python 3 function does
def handle_node(pos, node):
    'A node can either add B+W stones, play as B, or play as W.'
    props = node.properties
    black_stones_added = [coords.from_sgf(
        c) for c in props.get('AB', [])]
    white_stones_added = [coords.from_sgf(
        c) for c in props.get('AW', [])]
    if black_stones_added or white_stones_added:
        return add_stones(pos, black_stones_added, white_stones_added)
    # If B/W props are not present, then there is no move. But if it is present and equal to the empty string, then the move was a pass.
    elif 'B' in props:
        black_move = coords.from_sgf(props.get('B', [''])[0])
        return pos.play_move(black_move, color=go.BLACK)
    elif 'W' in props:
        white_move = coords.from_sgf(props.get('W', [''])[0])
        return pos.play_move(white_move, color=go.WHITE)
    else:
        return pos

Response: A node can either add B + W stones play as B or play as W.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def replay_sgf(sgf_contents):
    root_node = get_sgf_root_node(sgf_contents)
    props = root_node.properties
    assert int(sgf_prop(props.get('GM', ['1']))) == 1, "Not a Go SGF!"

    komi = 0
    if props.get('KM') is not None:
        komi = float(sgf_prop(props.get('KM')))
    result = utils.parse_game_result(sgf_prop(props.get('RE', '')))

    pos = Position(komi=komi)
    current_node = root_node
    while pos is not None and current_node.next is not None:
        pos = handle_node(pos, current_node)
        maybe_correct_next(pos, current_node.next)
        next_move = get_next_move(current_node)
        yield PositionWithContext(pos, next_move, result)
        current_node = current_node.next

Response: Wrapper for the replay_sgf function that returns a generator that yields PositionWithContext instances.


Instruction:
Implement a function in Python 3 to
apply the decoder to inputs given the context from the encoder.

Response:
def decode(self, inputs, context, inference=False):
        """
        Applies the decoder to inputs, given the context from the encoder.

        :param inputs: tensor with inputs (batch, seq_len) if 'batch_first'
            else (seq_len, batch)
        :param context: context from the encoder
        :param inference: if True inference mode, if False training mode
        """
        return self.decoder(inputs, context, inference)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def generate(self, inputs, context, beam_size):
        logits, scores, new_context = self.decode(inputs, context, True)
        logprobs = log_softmax(logits, dim=-1)
        logprobs, words = logprobs.topk(beam_size, dim=-1)
        return words, logprobs, scores, new_context

Response: Generates a new decoder from the given inputs and context.


Instruction:
Can you implement a function in Python 3 that
builds the eval_path for the current iteration and epoch.

Response:
def build_eval_path(self, epoch, iteration):
        """
        Appends index of the current epoch and index of the current iteration
        to the name of the file with results.

        :param epoch: index of the current epoch
        :param iteration: index of the current iteration
        """
        if iteration is not None:
            eval_fname = f'eval_epoch_{epoch}_iter_{iteration}'
        else:
            eval_fname = f'eval_epoch_{epoch}'
        eval_path = os.path.join(self.save_path, eval_fname)
        return eval_path


Instruction:
Can you tell what is the following Python 3 function doing
def run(self, calc_bleu=True, epoch=None, iteration=None, eval_path=None,
            summary=False, reference_path=None):
        """
        Runs translation on test dataset.

        :param calc_bleu: if True compares results with reference and computes
            BLEU score
        :param epoch: index of the current epoch
        :param iteration: index of the current iteration
        :param eval_path: path to the file for saving results
        :param summary: if True prints summary
        :param reference_path: path to the file with reference translation
        """
        if self.cuda:
            test_bleu = torch.cuda.FloatTensor([0])
            break_training = torch.cuda.LongTensor([0])
        else:
            test_bleu = torch.FloatTensor([0])
            break_training = torch.LongTensor([0])

        if eval_path is None:
            eval_path = self.build_eval_path(epoch, iteration)
        detok_eval_path = eval_path + '.detok'

        with contextlib.suppress(FileNotFoundError):
            os.remove(eval_path)
            os.remove(detok_eval_path)

        rank = get_rank()
        logging.info(f'Running evaluation on test set')
        self.model.eval()
        torch.cuda.empty_cache()

        output = self.evaluate(epoch, iteration, summary)
        output = output[:len(self.loader.dataset)]
        output = self.loader.dataset.unsort(output)

        if rank == 0:
            with open(eval_path, 'a') as eval_file:
                eval_file.writelines(output)
            if calc_bleu:
                self.run_detokenizer(eval_path)
                test_bleu[0] = self.run_sacrebleu(detok_eval_path, reference_path)
                if summary:
                    logging.info(f'BLEU on test dataset: {test_bleu[0]:.2f}')

                if self.target_bleu and test_bleu[0] >= self.target_bleu:
                    logging.info(f'Target accuracy reached')
                    break_training[0] = 1

        barrier()
        torch.cuda.empty_cache()
        logging.info(f'Finished evaluation on test set')

        if self.distributed:
            dist.broadcast(break_training, 0)
            dist.broadcast(test_bleu, 0)

        return test_bleu[0].item(), break_training[0].item()

Response: Runs translation on the current test set.


Instruction:
How would you code a function in Python 3 to
run evaluation on test dataset.

Response:
def evaluate(self, epoch, iteration, summary):
        """
        Runs evaluation on test dataset.

        :param epoch: index of the current epoch
        :param iteration: index of the current iteration
        :param summary: if True prints summary
        """
        batch_time = AverageMeter(False)
        tot_tok_per_sec = AverageMeter(False)
        iterations = AverageMeter(False)
        enc_seq_len = AverageMeter(False)
        dec_seq_len = AverageMeter(False)
        stats = {}

        output = []

        for i, (src, indices) in enumerate(self.loader):
            translate_timer = time.time()
            src, src_length = src

            batch_size = self.loader.batch_size
            global_batch_size = batch_size * get_world_size()
            beam_size = self.beam_size

            bos = [self.insert_target_start] * (batch_size * beam_size)
            bos = torch.LongTensor(bos)
            if self.batch_first:
                bos = bos.view(-1, 1)
            else:
                bos = bos.view(1, -1)

            src_length = torch.LongTensor(src_length)
            stats['total_enc_len'] = int(src_length.sum())

            if self.cuda:
                src = src.cuda()
                src_length = src_length.cuda()
                bos = bos.cuda()

            with torch.no_grad():
                context = self.model.encode(src, src_length)
                context = [context, src_length, None]

                if beam_size == 1:
                    generator = self.generator.greedy_search
                else:
                    generator = self.generator.beam_search
                preds, lengths, counter = generator(batch_size, bos, context)

            stats['total_dec_len'] = lengths.sum().item()
            stats['iters'] = counter

            indices = torch.tensor(indices).to(preds)
            preds = preds.scatter(0, indices.unsqueeze(1).expand_as(preds), preds)

            preds = gather_predictions(preds).cpu()

            for pred in preds:
                pred = pred.tolist()
                detok = self.tokenizer.detokenize(pred)
                output.append(detok + '\n')

            elapsed = time.time() - translate_timer
            batch_time.update(elapsed, batch_size)

            total_tokens = stats['total_dec_len'] + stats['total_enc_len']
            ttps = total_tokens / elapsed
            tot_tok_per_sec.update(ttps, batch_size)

            iterations.update(stats['iters'])
            enc_seq_len.update(stats['total_enc_len'] / batch_size, batch_size)
            dec_seq_len.update(stats['total_dec_len'] / batch_size, batch_size)

            if i % self.print_freq == 0:
                log = []
                log += f'TEST '
                if epoch is not None:
                    log += f'[{epoch}]'
                if iteration is not None:
                    log += f'[{iteration}]'
                log += f'[{i}/{len(self.loader)}]\t'
                log += f'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
                log += f'Decoder iters {iterations.val:.1f} ({iterations.avg:.1f})\t'
                log += f'Tok/s {tot_tok_per_sec.val:.0f} ({tot_tok_per_sec.avg:.0f})'
                log = ''.join(log)
                logging.info(log)

        tot_tok_per_sec.reduce('sum')
        enc_seq_len.reduce('mean')
        dec_seq_len.reduce('mean')
        batch_time.reduce('mean')
        iterations.reduce('sum')

        if summary and get_rank() == 0:
            time_per_sentence = (batch_time.avg / global_batch_size)
            log = []
            log += f'TEST SUMMARY:\n'
            log += f'Lines translated: {len(self.loader.dataset)}\t'
            log += f'Avg total tokens/s: {tot_tok_per_sec.avg:.0f}\n'
            log += f'Avg time per batch: {batch_time.avg:.3f} s\t'
            log += f'Avg time per sentence: {1000*time_per_sentence:.3f} ms\n'
            log += f'Avg encoder seq len: {enc_seq_len.avg:.2f}\t'
            log += f'Avg decoder seq len: {dec_seq_len.avg:.2f}\t'
            log += f'Total decoder iterations: {int(iterations.sum)}'
            log = ''.join(log)
            logging.info(log)

        return output


Instruction:
Write a Python 3 function that can
execute moses detokenizer on eval_path file and saves result to eval_path +. detok

Response:
def run_detokenizer(self, eval_path):
        """
        Executes moses detokenizer on eval_path file and saves result to
        eval_path + ".detok" file.

        :param eval_path: path to the tokenized input
        """
        logging.info('Running detokenizer')
        detok_path = os.path.join(self.dataset_dir, config.DETOKENIZER)
        detok_eval_path = eval_path + '.detok'

        with open(detok_eval_path, 'w') as detok_eval_file, \
                open(eval_path, 'r') as eval_file:
            subprocess.run(['perl', f'{detok_path}'], stdin=eval_file,
                           stdout=detok_eval_file, stderr=subprocess.DEVNULL)


Instruction:
Explain what the following Python 3 code does
def run_sacrebleu(self, detok_eval_path, reference_path):
        if reference_path is None:
            reference_path = os.path.join(self.dataset_dir,
                                          config.TGT_TEST_TARGET_FNAME)
        sacrebleu_params = '--score-only -lc --tokenize intl'
        logging.info(f'Running sacrebleu (parameters: {sacrebleu_params})')
        sacrebleu = subprocess.run([f'sacrebleu --input {detok_eval_path} \
                                    {reference_path} {sacrebleu_params}'],
                                   stdout=subprocess.PIPE, shell=True)
        test_bleu = float(sacrebleu.stdout.strip())
        return test_bleu

Response: Runs sacrebleu and returns BLEU score.


Instruction:
Can you generate the documentation for the following Python 3 function
def configure(project=LOGGING_PROJECT):
    if not project:
        sys.stderr.write('!! Error: The $LOGGING_PROJECT enviroment '
                         'variable is required in order to set up cloud logging. '
                         'Cloud logging is disabled.\n')
        return

    try:
        # if this fails, redirect stderr to /dev/null so no startup spam.
        with contextlib.redirect_stderr(io.StringIO()):
            client = glog.Client(project)
            client.setup_logging(logging.INFO)
    except:
        logging.basicConfig(level=logging.INFO)
        sys.stderr.write('!! Cloud logging disabled\n')

Response: Configures cloud logging with the given project.


Instruction:
Create a Python 3 function to
align the state_dict with the loaded_state_dict and updates the state_dict with the model_state_dict.

Response:
def align_and_update_state_dicts(model_state_dict, loaded_state_dict):
    """
    Strategy: suppose that the models that we will create will have prefixes appended
    to each of its keys, for example due to an extra level of nesting that the original
    pre-trained weights from ImageNet won't contain. For example, model.state_dict()
    might return backbone[0].body.res2.conv1.weight, while the pre-trained model contains
    res2.conv1.weight. We thus want to match both parameters together.
    For that, we look for each model weight, look among all loaded keys if there is one
    that is a suffix of the current weight name, and use it if that's the case.
    If multiple matches exist, take the one with longest size
    of the corresponding name. For example, for the same model as before, the pretrained
    weight file can contain both res2.conv1.weight, as well as conv1.weight. In this case,
    we want to match backbone[0].body.conv1.weight to conv1.weight, and
    backbone[0].body.res2.conv1.weight to res2.conv1.weight.
    """
    current_keys = sorted(list(model_state_dict.keys()))
    loaded_keys = sorted(list(loaded_state_dict.keys()))
    # get a matrix of string matches, where each (i, j) entry correspond to the size of the
    # loaded_key string, if it matches
    match_matrix = [
        len(j) if i.endswith(j) else 0 for i in current_keys for j in loaded_keys
    ]
    match_matrix = torch.as_tensor(match_matrix).view(
        len(current_keys), len(loaded_keys)
    )
    max_match_size, idxs = match_matrix.max(1)
    # remove indices that correspond to no-match
    idxs[max_match_size == 0] = -1

    # used for logging
    max_size = max([len(key) for key in current_keys]) if current_keys else 1
    max_size_loaded = max([len(key) for key in loaded_keys]) if loaded_keys else 1
    log_str_template = "{: <{}} loaded from {: <{}} of shape {}"
    logger = logging.getLogger(__name__)
    for idx_new, idx_old in enumerate(idxs.tolist()):
        if idx_old == -1:
            continue
        key = current_keys[idx_new]
        key_old = loaded_keys[idx_old]
        model_state_dict[key] = loaded_state_dict[key_old]
        logger.info(
            log_str_template.format(
                key,
                max_size,
                key_old,
                max_size_loaded,
                tuple(loaded_state_dict[key_old].shape),
            )
        )


Instruction:
Write a Python 3 script to
compute the ratio of gradient norm to weight norm.

Response:
def compute_update_ratio(weight_tensors, before_weights, after_weights):
    """Compute the ratio of gradient norm to weight norm."""
    deltas = [after - before for after,
              before in zip(after_weights, before_weights)]
    delta_norms = [np.linalg.norm(d.ravel()) for d in deltas]
    weight_norms = [np.linalg.norm(w.ravel()) for w in before_weights]
    ratios = [d / w for d, w in zip(delta_norms, weight_norms)]
    all_summaries = [
        tf.Summary.Value(tag='update_ratios/' +
                         tensor.name, simple_value=ratio)
        for tensor, ratio in zip(weight_tensors, ratios)]
    return tf.Summary(value=all_summaries)


Instruction:
Can you tell what is the following Python 3 function doing
def main(argv):
    tf_records = argv[1:]
    logging.info("Training on %s records: %s to %s",
                 len(tf_records), tf_records[0], tf_records[-1])
    with utils.logged_timer("Training"):
        train(*tf_records)
    if FLAGS.export_path:
        dual_net.export_model(FLAGS.export_path)
    if FLAGS.freeze:
        if FLAGS.use_tpu:
            dual_net.freeze_graph_tpu(FLAGS.export_path)
        else:
            dual_net.freeze_graph(FLAGS.export_path)

Response: Train on examples and export the updated model weights.


Instruction:
Can you generate the documentation for the following Python 3 function
def main(unused_argv):
    if FLAGS.use_tpu:
        dual_net.freeze_graph_tpu(FLAGS.model_path)
    else:
        dual_net.freeze_graph(FLAGS.model_path)

Response: Freeze a model to a GraphDef proto.


Instruction:
Can you generate the documentation for the following Python 3 function
def grouper(n, iterable):
    return (iterable[i:i + n] for i in range(0, len(iterable), n))

Response: Returns a list of n elements from an iterable.


Instruction:
Implement a function in Python 3 to
sample num_positions postions from each game in sgf_dir

Response:
def subsample():
    """Sample num_positions postions from each game in sgf_dir

    Usage:
        python3 sharp_positions.py subsample --num_positions 10 --sgf_dir data/s

    NOTE(sethtroisi): see link for a script to truncate SGFs at move number
        https://github.com/sethtroisi/go-scripts
    """

    sgf_files = oneoff_utils.find_and_filter_sgf_files(
        FLAGS.sgf_dir, None, None)

    with open(FLAG.collection, 'w') as collection:
        fails = 0
        for path in tqdm(sorted(sgf_files)):
            try:
                positions, moves, results = oneoff_utils.parse_sgf(path)
            except KeyboardInterrupt:
                raise
            except Exception as e:
                fails += 1
                print("Fail {}, while parsing {}: {}".format(fails, path, e))
                continue

            moves = len(positions)
            indexes = random.sample(range(10, moves), FLAGS.num_positions)
            for index in sorted(indexes):
                collection.write('{}, {}\n'.format(path, index))


Instruction:
Make a summary of the following Python 3 code
def evaluate():

    def short_str(v):
        if isinstance(v, float):
            return "{.3f}".format(v)
        return str(v)

    # Load positons
    sgf_names, all_positions = get_final_positions()

    # Run and save some data about each position
    # Save to csv because that's easy
    model_paths = oneoff_utils.get_model_paths(FLAGS.model_dir)
    num_models = len(model_paths)
    print("Evaluating {} models: {} to {}".format(
        num_models, model_paths[0], model_paths[-1]))
    print()

    with open(FLAGS.results, "w") as results:
        results.write(",".join(sgf_names) + "\n")

        player = None
        for idx in tqdm(range(FLAGS.min_idx, num_models, 1), desc="model"):
            model = model_paths[idx]

            if player and idx % 50 == 0:
                player.network.sess.close()
                tf.reset_default_graph()
                player = None

            if player:
                oneoff_utils.restore_params(model, player)
            else:
                player = oneoff_utils.load_player(model)

            row = [model]
            for positions in grouper(FLAGS.batch_size, all_positions):
                probs, values = player.network.run_many(positions)
                # NOTE(sethtroisi): For now we store the top n moves to shrink
                # the size of the recorded data.

                top_n = FLAGS.top_n
                top_policy_move = np.fliplr(np.argsort(probs))[:,:top_n]
                top_policy_value = np.fliplr(np.sort(probs))[:,:top_n]

                # One position at a time
                for v, m, p in zip(values, top_policy_move, top_policy_value):
                    row.append(v)
                    row.extend(itertools.chain.from_iterable(zip(m, p)))

                if len(positions) > 10:
                    average_seen = top_policy_value.sum() / len(positions)
                    if average_seen < 0.3:
                        print("\t", average_seen, top_policy_value.sum(axis=-1))

            results.write(",".join(map(short_str, row)) + "\n")

Response: Evaluate the policy and value for each position


Instruction:
How would you implement a function in Python 3 that
finds a subset of problems that maximal explains rating.

Response:
def minimize():
    """Find a subset of problems that maximal explains rating.

    Usage:
        python3 sharp_positions.py minimize \
            --model_dir models --sgf_dir data/s
            --rating_json ratings.json --results results.csv
    """
    ########################### HYPER PARAMETERS ###############################

    # Stop when r2 is this much worse than full set of positions
    r2_stopping_percent = 0.96
    # for this many iterations
    stopping_iterations = 5

    # Limit SVM to a smaller number of positions to speed up code.
    max_positions_fit = 300
    # Filter any position that "contributes" less than this percent of max.
    filter_contribution_percent = 0.3
    # Never filter more than this many positions in one iterations
    filter_limit = 25

    ########################### HYPER PARAMETERS ###############################

    # Load positons
    model_paths = oneoff_utils.get_model_paths(FLAGS.model_dir)
    num_models = len(model_paths)
    assert num_models > 0, FLAGS.model_dir

    # Load model ratings
    # wget https://cloudygo.com/v12-19x19/json/ratings.json
    ratings = json.load(open(FLAGS.rating_json))
    raw_ratings = {int(r[0]): float(r[1]) for r in ratings}

    model_ratings = []
    for model in model_paths:
        model_idx = get_model_idx(model)
        if model_idx < FLAGS.min_idx:
            continue

        model_ratings.append(raw_ratings[model_idx])
    model_ratings = np.array(model_ratings)

    assert 0 < len(model_ratings) <= num_models, len(model_ratings)
    num_models = len(model_ratings)

    sgf_names, all_positions = get_final_positions()
    # Trim off common path prefix.
    common_path = os.path.commonpath(sgf_names)
    sgf_names = [name[len(common_path) + 1:] for name in sgf_names]

    print("Considering {} positions, {} models".format(
        len(all_positions), num_models))
    print()

    # Load model data
    top_n = FLAGS.top_n
    positions = defaultdict(list)
    with open(FLAGS.results) as results:
        headers = results.readline().strip()
        assert headers.count(",") + 1 == len(sgf_names)

        # Row is <model_name> + positions x [value, top_n x [move, move_policy]]
        for row in tqdm(results.readlines(), desc="result line"):
            data = row.split(",")
            model_idx = get_model_idx(data.pop(0))
            if model_idx < FLAGS.min_idx:
                continue

            data_per = 1 + top_n * 2
            assert len(data) % data_per == 0, len(data)

            for position, position_data in enumerate(grouper(data_per, data)):
                value = float(position_data.pop(0))
                moves = list(map(int, position_data[0::2]))
                move_policy = list(map(float, position_data[1::2]))

                positions[position].append([value, moves, move_policy])

    def one_hot(n, i):
        one_hot = [0] * n
        if 0 <= i < n:
            one_hot[i] += 1
        return one_hot

    # NOTE: top_n isn't the same semantic value here and can be increased.
    one_hot_moves = top_n
    num_features = 1 + 5 + (one_hot_moves + 1)

    # Features by position
    features = []
    pos_top_moves = []
    for position, data in tqdm(positions.items(), desc="featurize"):
        assert len(data) == num_models, len(data)

        top_moves = Counter([d[1][0] for d in data])
        top_n_moves = [m for m, c in top_moves.most_common(one_hot_moves)]
        if len(top_n_moves) < one_hot_moves:
            top_n_moves.extend([-1] * (one_hot_moves - len(top_n_moves)))
        assert len(top_n_moves) == one_hot_moves, "pad with dummy moves"
        pos_top_moves.append(top_n_moves)

        # Eventaully we want
        # [model 1 position 1 features, m1 p2 features, m1 p3 features, ... ]
        # [model 2 position 1 features, m2 p2 features, m2 p3 features, ... ]
        # [model 3 position 1 features, m3 p2 features, m3 p3 features, ... ]
        # ...
        # [model m position 1 features, mm p2 features, mm p3 features, ... ]

        # We'll do position selection by joining [model x position_feature]

        feature_columns = []
        for model, (v, m, mv) in enumerate(data):
            # Featurization (for each positions):
            #   * Value (-1 to 1), Bucketed value
            #   * Cluster all model by top_n moves (X,Y,Z or other)?
            #     * value of that move for model
            #   * policy value of top move
            model_features = []

            model_features.append(2 * v - 1)
            # NOTE(sethtroisi): Consider bucketize value by value percentiles.
            value_bucket = np.searchsorted((0.2, 0.4, 0.6, 0.8), v)
            model_features.extend(one_hot(5, value_bucket))

            # Policy weight for most common X moves (among all models).
            policy_weights = [0] * (one_hot_moves + 1)
            for move, policy_value in zip(m, mv):
                if move in top_n_moves:
                    policy_weights[top_n_moves.index(move)] = policy_value
                else:
                    policy_weights[-1] += policy_value
            model_features.extend(policy_weights)

            assert len(model_features) == num_features

            feature_columns.append(model_features)
        features.append(feature_columns)

    features = np.array(features)
    print("Feature shape", features.shape)
    print()

    # Split the models to test / train
    train_size = int(num_models * 0.9)
    train_models = sorted(np.random.permutation(num_models)[:train_size])
    test_models = sorted(set(range(num_models)) - set(train_models))
    assert set(train_models + test_models) == set(range(num_models))
    features_train = features[:, train_models, :]
    features_test  = features[:, test_models, :]

    labels_train = model_ratings[train_models]
    labels_test = model_ratings[test_models]

    # Choose some set of positions and see how well they explain ratings
    positions_to_use = set(positions.keys())
    linearSVM = svm.LinearSVR()
    best_test_r2 = 0
    below_threshold = 0

    for iteration in itertools.count(1):
        iter_positions = np.random.permutation(list(positions_to_use))
        iter_positions = sorted(iter_positions[:max_positions_fit])

        # Take this set of positions and build X
        X = np.concatenate(features_train[iter_positions], axis=1)
        Xtest = np.concatenate(features_test[iter_positions], axis=1)
        assert X.shape == (train_size, num_features * len(iter_positions))

        linearSVM.fit(X, labels_train)

        score_train = linearSVM.score(X, labels_train)
        score_test = linearSVM.score(Xtest, labels_test)
        print("iter {}, {}/{} included, R^2: {:.4f} train, {:.3f} test".format(
            iteration, len(iter_positions), len(positions_to_use),
            score_train, score_test))

        # Determine the most and least useful position:
        # TODO(amj,brilee): Validate this math.
        assert len(linearSVM.coef_) == num_features * len(iter_positions)

        # The intercepts tell us how much this contributes to overall rating
        # but coef tell us how much different answers differentiate rating.
        coef_groups = list(grouper(num_features, linearSVM.coef_))
        position_coefs = [abs(sum(c)) for c in coef_groups]

        pos_value_idx = np.argsort(position_coefs)
        max_pos = pos_value_idx[-1]
        most_value = position_coefs[max_pos]

        print("\tMost value {} => {:.1f} {}".format(
            max_pos, most_value, sgf_names[iter_positions[max_pos]]))

        # Drop any positions that aren't very useful
        for dropped, pos_idx in enumerate(pos_value_idx[:filter_limit], 1):
            contribution = position_coefs[pos_idx]
            positions_to_use.remove(iter_positions[pos_idx])
            print("\t\tdropping({}): {:.1f} {}".format(
                dropped, contribution, sgf_names[iter_positions[pos_idx]]))

            if contribution > filter_contribution_percent * most_value:
                break
        print()

        best_test_r2 = max(best_test_r2, score_test)
        if score_test > r2_stopping_percent * best_test_r2:
            below_threshold = 0
        else:
            below_threshold += 1
            if below_threshold == stopping_iterations:
                print("{}% decrease in R^2, stopping".format(
                    100 - int(100 * r2_stopping_percent)))
                break

    # Write down the differentiating positions and their answers.
    svm_data = []
    for position_idx in list(reversed(pos_value_idx)):
        coefs = coef_groups[position_idx]

        # Global position index.
        position = iter_positions[position_idx]
        sgf_name = sgf_names[position]
        top_moves = pos_top_moves[position]

        svm_data.append([sgf_name, [top_moves, coefs.tolist()]])

    with open(FLAGS.SVM_json, "w") as svm_json:
        json.dump(svm_data, svm_json)
    print("Dumped data about {} positions to {}".format(
        len(svm_data), FLAGS.SVM_json))


Instruction:
Create a Python 3 function for
downloading content from a url.

Response:
def download_from_url(path, url):
  """Download content from a url.

  Args:
    path: string directory where file will be downloaded
    url: string url

  Returns:
    Full path to downloaded file
  """
  filename = url.split("/")[-1]
  found_file = find_file(path, filename, max_depth=0)
  if found_file is None:
    filename = os.path.join(path, filename)
    tf.logging.info("Downloading from %s to %s." % (url, filename))
    inprogress_filepath = filename + ".incomplete"
    inprogress_filepath, _ = urllib.request.urlretrieve(
        url, inprogress_filepath, reporthook=download_report_hook)
    # Print newline to clear the carriage return from the download progress.
    print()
    tf.gfile.Rename(inprogress_filepath, filename)
    return filename
  else:
    tf.logging.info("Already downloaded: %s (at %s)." % (url, found_file))
    return found_file


Instruction:
Write a Python 3 script to
give segmentation masks and the bounding boxes corresponding to the location of the masks in the image, this function crops and resizes the masks in the position defined by the boxes. This prepares the masks for them to be fed to the loss computation as the targets. Arguments: segmentation_masks: an instance of SegmentationMask proposals: an instance of BoxList

Response:
def project_masks_on_boxes(segmentation_masks, proposals, discretization_size):
    """
    Given segmentation masks and the bounding boxes corresponding
    to the location of the masks in the image, this function
    crops and resizes the masks in the position defined by the
    boxes. This prepares the masks for them to be fed to the
    loss computation as the targets.

    Arguments:
        segmentation_masks: an instance of SegmentationMask
        proposals: an instance of BoxList
    """
    masks = []
    M = discretization_size
    device = proposals.bbox.device
    proposals = proposals.convert("xyxy")
    assert segmentation_masks.size == proposals.size, "{}, {}".format(
        segmentation_masks, proposals
    )
    # TODO put the proposals on the CPU, as the representation for the
    # masks is not efficient GPU-wise (possibly several small tensors for
    # representing a single instance mask)
    proposals = proposals.bbox.to(torch.device("cpu"))
    for segmentation_mask, proposal in zip(segmentation_masks, proposals):
        # crop the masks, resize them to the desired resolution and
        # then convert them to the tensor representation,
        # instead of the list representation that was used
        cropped_mask = segmentation_mask.crop(proposal)
        scaled_mask = cropped_mask.resize((M, M))
        mask = scaled_mask.convert(mode="mask")
        masks.append(mask)
    if len(masks) == 0:
        return torch.empty(0, dtype=torch.float32, device=device)
    return torch.stack(masks, dim=0).to(device, dtype=torch.float32)


Instruction:
Create a Python 3 function to
evaluate detection proposal recall metrics.

Response:
def evaluate_box_proposals(
    predictions, dataset, thresholds=None, area="all", limit=None
):
    """Evaluate detection proposal recall metrics. This function is a much
    faster alternative to the official COCO API recall evaluation code. However,
    it produces slightly different results.
    """
    # Record max overlap value for each gt box
    # Return vector of overlap values
    areas = {
        "all": 0,
        "small": 1,
        "medium": 2,
        "large": 3,
        "96-128": 4,
        "128-256": 5,
        "256-512": 6,
        "512-inf": 7,
    }
    area_ranges = [
        [0 ** 2, 1e5 ** 2],  # all
        [0 ** 2, 32 ** 2],  # small
        [32 ** 2, 96 ** 2],  # medium
        [96 ** 2, 1e5 ** 2],  # large
        [96 ** 2, 128 ** 2],  # 96-128
        [128 ** 2, 256 ** 2],  # 128-256
        [256 ** 2, 512 ** 2],  # 256-512
        [512 ** 2, 1e5 ** 2],
    ]  # 512-inf
    assert area in areas, "Unknown area range: {}".format(area)
    area_range = area_ranges[areas[area]]
    gt_overlaps = []
    num_pos = 0

    for image_id, prediction in enumerate(predictions):
        original_id = dataset.id_to_img_map[image_id]

        img_info = dataset.get_img_info(image_id)
        image_width = img_info["width"]
        image_height = img_info["height"]
        prediction = prediction.resize((image_width, image_height))

        # sort predictions in descending order
        # TODO maybe remove this and make it explicit in the documentation
        inds = prediction.get_field("objectness").sort(descending=True)[1]
        prediction = prediction[inds]

        ann_ids = dataset.coco.getAnnIds(imgIds=original_id)
        anno = dataset.coco.loadAnns(ann_ids)
        gt_boxes = [obj["bbox"] for obj in anno if obj["iscrowd"] == 0]
        gt_boxes = torch.as_tensor(gt_boxes).reshape(-1, 4)  # guard against no boxes
        gt_boxes = BoxList(gt_boxes, (image_width, image_height), mode="xywh").convert(
            "xyxy"
        )
        gt_areas = torch.as_tensor([obj["area"] for obj in anno if obj["iscrowd"] == 0])

        if len(gt_boxes) == 0:
            continue

        valid_gt_inds = (gt_areas >= area_range[0]) & (gt_areas <= area_range[1])
        gt_boxes = gt_boxes[valid_gt_inds]

        num_pos += len(gt_boxes)

        if len(gt_boxes) == 0:
            continue

        if len(prediction) == 0:
            continue

        if limit is not None and len(prediction) > limit:
            prediction = prediction[:limit]

        overlaps = boxlist_iou(prediction, gt_boxes)

        _gt_overlaps = torch.zeros(len(gt_boxes))
        for j in range(min(len(prediction), len(gt_boxes))):
            # find which proposal box maximally covers each gt box
            # and get the iou amount of coverage for each gt box
            max_overlaps, argmax_overlaps = overlaps.max(dim=0)

            # find which gt box is 'best' covered (i.e. 'best' = most iou)
            gt_ovr, gt_ind = max_overlaps.max(dim=0)
            assert gt_ovr >= 0
            # find the proposal box that covers the best covered gt box
            box_ind = argmax_overlaps[gt_ind]
            # record the iou coverage of this gt box
            _gt_overlaps[j] = overlaps[box_ind, gt_ind]
            assert _gt_overlaps[j] == gt_ovr
            # mark the proposal box and the gt box as used
            overlaps[box_ind, :] = -1
            overlaps[:, gt_ind] = -1

        # append recorded iou coverage level
        gt_overlaps.append(_gt_overlaps)
    gt_overlaps = torch.cat(gt_overlaps, dim=0)
    gt_overlaps, _ = torch.sort(gt_overlaps)

    if thresholds is None:
        step = 0.05
        thresholds = torch.arange(0.5, 0.95 + 1e-5, step, dtype=torch.float32)
    recalls = torch.zeros_like(thresholds)
    # compute recall for each iou threshold
    for i, t in enumerate(thresholds):
        recalls[i] = (gt_overlaps >= t).float().sum() / float(num_pos)
    # ar = 2 * np.trapz(recalls, thresholds)
    ar = recalls.mean()
    return {
        "ar": ar,
        "recalls": recalls,
        "thresholds": thresholds,
        "gt_overlaps": gt_overlaps,
        "num_pos": num_pos,
    }


Instruction:
Can you generate the documentation for the following Python 3 function
def _decode_crop_and_flip(image_buffer, num_channels):
  # A large fraction of image datasets contain a human-annotated bounding box
  # delineating the region of the image containing the object of interest.  We
  # choose to create a new bounding box for the object which is a randomly
  # distorted version of the human-annotated bounding box that obeys an
  # allowed range of aspect ratios, sizes and overlap with the human-annotated
  # bounding box. If no box is supplied, then we assume the bounding box is
  # the entire image.

  min_object_covered=0.1
  aspect_ratio_range=[0.75, 1.33]
  area_range=[0.05, 1.0]
  max_attempts=100

  mlperf_log.resnet_print(key=mlperf_log.INPUT_DISTORTED_CROP_MIN_OBJ_COV,
                          value=min_object_covered)
  mlperf_log.resnet_print(key=mlperf_log.INPUT_DISTORTED_CROP_RATIO_RANGE,
                          value=aspect_ratio_range)
  mlperf_log.resnet_print(key=mlperf_log.INPUT_DISTORTED_CROP_AREA_RANGE,
                          value=area_range)
  mlperf_log.resnet_print(key=mlperf_log.INPUT_DISTORTED_CROP_MAX_ATTEMPTS,
                          value=max_attempts)
  mlperf_log.resnet_print(key=mlperf_log.INPUT_CROP_USES_BBOXES, value=False)

  bbox = tf.constant([0.0, 0.0, 1.0, 1.0],
                     dtype=tf.float32, shape=[1, 1, 4])   #From the entire image
  sample_distorted_bounding_box = tf.image.sample_distorted_bounding_box(
      tf.image.extract_jpeg_shape(image_buffer),
      bounding_boxes=bbox,
      min_object_covered=min_object_covered,
      aspect_ratio_range=aspect_ratio_range,
      area_range=area_range,
      max_attempts=max_attempts,
      use_image_if_no_bounding_boxes=True)
  bbox_begin, bbox_size, _ = sample_distorted_bounding_box

  # Reassemble the bounding box in the format the crop op requires.
  offset_y, offset_x, _ = tf.unstack(bbox_begin)
  target_height, target_width, _ = tf.unstack(bbox_size)
  crop_window = tf.stack([offset_y, offset_x, target_height, target_width])

  # Use the fused decode and crop op here, which is faster than each in series.
  cropped = tf.image.decode_and_crop_jpeg(
      image_buffer, crop_window, channels=num_channels)

  # Flip to add a little more random distortion in.
  mlperf_log.resnet_print(key=mlperf_log.INPUT_RANDOM_FLIP)
  cropped = tf.image.random_flip_left_right(cropped)
  return cropped

Response: This function is used to decode a given image buffer and crop it to a random part of the image and randomly flips it.


Instruction:
Can you write a function in Python 3 where it
performs central crop of the given image list.

Response:
def _central_crop(image, crop_height, crop_width):
  """Performs central crops of the given image list.

  Args:
    image: a 3-D image tensor
    crop_height: the height of the image following the crop.
    crop_width: the width of the image following the crop.

  Returns:
    3-D tensor with cropped image.
  """
  shape = tf.shape(image)
  height, width = shape[0], shape[1]

  mlperf_log.resnet_print(key=mlperf_log.INPUT_CENTRAL_CROP,
                          value=[crop_height, crop_width])

  amount_to_be_cropped_h = (height - crop_height)
  crop_top = amount_to_be_cropped_h // 2
  amount_to_be_cropped_w = (width - crop_width)
  crop_left = amount_to_be_cropped_w // 2
  return tf.slice(
      image, [crop_top, crop_left, 0], [crop_height, crop_width, -1])


Instruction:
Can you tell what is the following Python 3 function doing
def _mean_image_subtraction(image, means, num_channels):
  if image.get_shape().ndims != 3:
    raise ValueError('Input must be of size [height, width, C>0]')

  if len(means) != num_channels:
    raise ValueError('len(means) must match the number of channels')

  mlperf_log.resnet_print(key=mlperf_log.INPUT_MEAN_SUBTRACTION,
                          value=means)

  # We have a 1-D tensor of means; convert to 3-D.
  means = tf.expand_dims(tf.expand_dims(means, 0), 0)

  return image - means

Response: Subtracts the given means from each image channel.


Instruction:
Explain what the following Python 3 code does
def _smallest_size_at_least(height, width, resize_min):
  resize_min = tf.cast(resize_min, tf.float32)

  # Convert to floats to make subsequent calculations go smoothly.
  height, width = tf.cast(height, tf.float32), tf.cast(width, tf.float32)

  smaller_dim = tf.minimum(height, width)
  scale_ratio = resize_min / smaller_dim

  # Convert back to ints to make heights and widths that TF ops will accept.
  new_height = tf.cast(height * scale_ratio, tf.int32)
  new_width = tf.cast(width * scale_ratio, tf.int32)

  return new_height, new_width

Response: Computes new shape with the smallest side equal to smallest_side.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _aspect_preserving_resize(image, resize_min):
  mlperf_log.resnet_print(key=mlperf_log.INPUT_RESIZE_ASPECT_PRESERVING,
                          value={"min": resize_min})

  shape = tf.shape(image)
  height, width = shape[0], shape[1]

  new_height, new_width = _smallest_size_at_least(height, width, resize_min)

  return _resize_image(image, new_height, new_width)

Response: Resize images preserving the original aspect ratio.


Instruction:
Can you create a Python 3 function that
searches for sequence of subtoken ids with the largest probability.

Response:
def sequence_beam_search(
    symbols_to_logits_fn, initial_ids, initial_cache, vocab_size, beam_size,
    alpha, max_decode_length, eos_id):
  """Search for sequence of subtoken ids with the largest probability.

  Args:
    symbols_to_logits_fn: A function that takes in ids, index, and cache as
      arguments. The passed in arguments will have shape:
        ids -> [batch_size * beam_size, index]
        index -> [] (scalar)
        cache -> nested dictionary of tensors [batch_size * beam_size, ...]
      The function must return logits and new cache.
        logits -> [batch * beam_size, vocab_size]
        new cache -> same shape/structure as inputted cache
    initial_ids: Starting ids for each batch item.
      int32 tensor with shape [batch_size]
    initial_cache: dict containing starting decoder variables information
    vocab_size: int size of tokens
    beam_size: int number of beams
    alpha: float defining the strength of length normalization
    max_decode_length: maximum length to decoded sequence
    eos_id: int id of eos token, used to determine when a sequence has finished

  Returns:
    Top decoded sequences [batch_size, beam_size, max_decode_length]
    sequence scores [batch_size, beam_size]
  """
  batch_size = tf.shape(initial_ids)[0]
  sbs = SequenceBeamSearch(symbols_to_logits_fn, vocab_size, batch_size,
                           beam_size, alpha, max_decode_length, eos_id)
  return sbs.search(initial_ids, initial_cache)


Instruction:
How would you implement a function in Python 3 that
returns a list of the tensor s shape and ensure no None values in list.

Response:
def _shape_list(tensor):
  """Return a list of the tensor's shape, and ensure no None values in list."""
  # Get statically known shape (may contain None's for unknown dimensions)
  shape = tensor.get_shape().as_list()

  # Ensure that the shape values are not None
  dynamic_shape = tf.shape(tensor)
  for i in range(len(shape)):
    if shape[i] is None:
      shape[i] = dynamic_shape[i]
  return shape


Instruction:
Given the following Python 3 function, write the documentation
def _unflatten_beam_dim(tensor, batch_size, beam_size):
  shape = _shape_list(tensor)
  new_shape = [batch_size, beam_size] + shape[1:]
  return tf.reshape(tensor, new_shape)

Response: Reshapes first dimension back to [ batch_size beam_size... )


Instruction:
How would you explain what the following Python 3 function does
def _gather_beams(nested, beam_indices, batch_size, new_beam_size):
  # Computes the i'th coodinate that contains the batch index for gather_nd.
  # Batch pos is a tensor like [[0,0,0,0,],[1,1,1,1],..].
  batch_pos = tf.range(batch_size * new_beam_size) // new_beam_size
  batch_pos = tf.reshape(batch_pos, [batch_size, new_beam_size])

  # Create coordinates to be passed to tf.gather_nd. Stacking creates a tensor
  # with shape [batch_size, beam_size, 2], where the last dimension contains
  # the (i, j) gathering coordinates.
  coordinates = tf.stack([batch_pos, beam_indices], axis=2)

  return nest.map_structure(
      lambda state: tf.gather_nd(state, coordinates), nested)

Response: This function is used to gather beams from nested structure of tensors.


Instruction:
Can you write a function in Python 3 where it
gathers top k beams from nested structure.

Response:
def _gather_topk_beams(nested, score_or_log_prob, batch_size, beam_size):
  """Gather top beams from nested structure."""
  _, topk_indexes = tf.nn.top_k(score_or_log_prob, k=beam_size)
  return _gather_beams(nested, topk_indexes, batch_size, beam_size)


Instruction:
How would you explain what the following Python 3 function does
def search(self, initial_ids, initial_cache):
    state, state_shapes = self._create_initial_state(initial_ids, initial_cache)

    finished_state = tf.while_loop(
        self._continue_search, self._search_step, loop_vars=[state],
        shape_invariants=[state_shapes], parallel_iterations=1, back_prop=False)
    finished_state = finished_state[0]

    alive_seq = finished_state[_StateKeys.ALIVE_SEQ]
    alive_log_probs = finished_state[_StateKeys.ALIVE_LOG_PROBS]
    finished_seq = finished_state[_StateKeys.FINISHED_SEQ]
    finished_scores = finished_state[_StateKeys.FINISHED_SCORES]
    finished_flags = finished_state[_StateKeys.FINISHED_FLAGS]

    # Account for corner case where there are no finished sequences for a
    # particular batch item. In that case, return alive sequences for that batch
    # item.
    finished_seq = tf.where(
        tf.reduce_any(finished_flags, 1), finished_seq, alive_seq)
    finished_scores = tf.where(
        tf.reduce_any(finished_flags, 1), finished_scores, alive_log_probs)
    return finished_seq, finished_scores

Response: Beam search for sequences with highest scores.


Instruction:
Write a Python 3 script for
creating the initial state dictionary and its shape invariants.

Response:
def _create_initial_state(self, initial_ids, initial_cache):
    """Return initial state dictionary and its shape invariants.

    Args:
      initial_ids: initial ids to pass into the symbols_to_logits_fn.
        int tensor with shape [batch_size, 1]
      initial_cache: dictionary storing values to be passed into the
        symbols_to_logits_fn.

    Returns:
        state and shape invariant dictionaries with keys from _StateKeys
    """
    # Current loop index (starts at 0)
    cur_index = tf.constant(0)

    # Create alive sequence with shape [batch_size, beam_size, 1]
    alive_seq = _expand_to_beam_size(initial_ids, self.beam_size)
    alive_seq = tf.expand_dims(alive_seq, axis=2)

    # Create tensor for storing initial log probabilities.
    # Assume initial_ids are prob 1.0
    initial_log_probs = tf.constant(
        [[0.] + [-float("inf")] * (self.beam_size - 1)])
    alive_log_probs = tf.tile(initial_log_probs, [self.batch_size, 1])

    # Expand all values stored in the dictionary to the beam size, so that each
    # beam has a separate cache.
    alive_cache = nest.map_structure(
        lambda t: _expand_to_beam_size(t, self.beam_size), initial_cache)

    # Initialize tensor storing finished sequences with filler values.
    finished_seq = tf.zeros(tf.shape(alive_seq), tf.int32)

    # Set scores of the initial finished seqs to negative infinity.
    finished_scores = tf.ones([self.batch_size, self.beam_size]) * -INF

    # Initialize finished flags with all False values.
    finished_flags = tf.zeros([self.batch_size, self.beam_size], tf.bool)

    # Create state dictionary
    state = {
        _StateKeys.CUR_INDEX: cur_index,
        _StateKeys.ALIVE_SEQ: alive_seq,
        _StateKeys.ALIVE_LOG_PROBS: alive_log_probs,
        _StateKeys.ALIVE_CACHE: alive_cache,
        _StateKeys.FINISHED_SEQ: finished_seq,
        _StateKeys.FINISHED_SCORES: finished_scores,
        _StateKeys.FINISHED_FLAGS: finished_flags
    }

    # Create state invariants for each value in the state dictionary. Each
    # dimension must be a constant or None. A None dimension means either:
    #   1) the dimension's value is a tensor that remains the same but may
    #      depend on the input sequence to the model (e.g. batch size).
    #   2) the dimension may have different values on different iterations.
    state_shape_invariants = {
        _StateKeys.CUR_INDEX: tf.TensorShape([]),
        _StateKeys.ALIVE_SEQ: tf.TensorShape([None, self.beam_size, None]),
        _StateKeys.ALIVE_LOG_PROBS: tf.TensorShape([None, self.beam_size]),
        _StateKeys.ALIVE_CACHE: nest.map_structure(
            _get_shape_keep_last_dim, alive_cache),
        _StateKeys.FINISHED_SEQ: tf.TensorShape([None, self.beam_size, None]),
        _StateKeys.FINISHED_SCORES: tf.TensorShape([None, self.beam_size]),
        _StateKeys.FINISHED_FLAGS: tf.TensorShape([None, self.beam_size])
    }

    return state, state_shape_invariants


Instruction:
Can you generate the documentation for the following Python 3 function
def _continue_search(self, state):
    i = state[_StateKeys.CUR_INDEX]
    alive_log_probs = state[_StateKeys.ALIVE_LOG_PROBS]
    finished_scores = state[_StateKeys.FINISHED_SCORES]
    finished_flags = state[_StateKeys.FINISHED_FLAGS]

    not_at_max_decode_length = tf.less(i, self.max_decode_length)

    # Calculate largest length penalty (the larger penalty, the better score).
    max_length_norm = _length_normalization(self.alpha, self.max_decode_length)
    # Get the best possible scores from alive sequences.
    best_alive_scores = alive_log_probs[:, 0] / max_length_norm

    # Compute worst score in finished sequences for each batch element
    finished_scores *= tf.to_float(finished_flags)  # set filler scores to zero
    lowest_finished_scores = tf.reduce_min(finished_scores, axis=1)

    # If there are no finished sequences in a batch element, then set the lowest
    # finished score to -INF for that element.
    finished_batches = tf.reduce_any(finished_flags, 1)
    lowest_finished_scores += (1. - tf.to_float(finished_batches)) * -INF

    worst_finished_score_better_than_best_alive_score = tf.reduce_all(
        tf.greater(lowest_finished_scores, best_alive_scores)
    )

    return tf.logical_and(
        not_at_max_decode_length,
        tf.logical_not(worst_finished_score_better_than_best_alive_score)
    )

Response: Return whether to continue the search loop.


Instruction:
Write a Python 3 script for
beaming search loop body.

Response:
def _search_step(self, state):
    """Beam search loop body.

    Grow alive sequences by a single ID. Sequences that have reached the EOS
    token are marked as finished. The alive and finished sequences with the
    highest log probabilities and scores are returned.

    A sequence's finished score is calculating by dividing the log probability
    by the length normalization factor. Without length normalization, the
    search is more likely to return shorter sequences.

    Args:
      state: A dictionary with the current loop state.

    Returns:
      new state dictionary.
    """
    # Grow alive sequences by one token.
    new_seq, new_log_probs, new_cache = self._grow_alive_seq(state)
    # Collect top beam_size alive sequences
    alive_state = self._get_new_alive_state(new_seq, new_log_probs, new_cache)

    # Combine newly finished sequences with existing finished sequences, and
    # collect the top k scoring sequences.
    finished_state = self._get_new_finished_state(state, new_seq, new_log_probs)

    # Increment loop index and create new state dictionary
    new_state = {_StateKeys.CUR_INDEX: state[_StateKeys.CUR_INDEX] + 1}
    new_state.update(alive_state)
    new_state.update(finished_state)
    return [new_state]