Instruction:
Given the following Python 3 function, write the documentation
def validate(cls, mapper_spec):
    if mapper_spec.input_reader_class() != cls:
      raise errors.BadReaderParamsError("Input reader class mismatch")

    params = _get_params(mapper_spec, allowed_keys=cls._PARAMS)
    if (cls.VERSION_IDS_PARAM not in params and
        cls.MODULE_VERSIONS_PARAM not in params):
      raise errors.BadReaderParamsError("Must specify a list of version ids or "
                                        "module/version ids for mapper input")
    if (cls.VERSION_IDS_PARAM in params and
        cls.MODULE_VERSIONS_PARAM in params):
      raise errors.BadReaderParamsError("Can not supply both version ids or "
                                        "module/version ids. Use only one.")
    if (cls.START_TIME_PARAM not in params or
        params[cls.START_TIME_PARAM] is None):
      raise errors.BadReaderParamsError("Must specify a starting time for "
                                        "mapper input")
    if cls.END_TIME_PARAM not in params or params[cls.END_TIME_PARAM] is None:
      params[cls.END_TIME_PARAM] = time.time()

    if params[cls.START_TIME_PARAM] >= params[cls.END_TIME_PARAM]:
      raise errors.BadReaderParamsError("The starting time cannot be later "
                                        "than or the same as the ending time.")

    if cls._PROTOTYPE_REQUEST_PARAM in params:
      try:
        params[cls._PROTOTYPE_REQUEST_PARAM] = log_service_pb.LogReadRequest(
            params[cls._PROTOTYPE_REQUEST_PARAM])
      except (TypeError, ProtocolBuffer.ProtocolBufferDecodeError):
        raise errors.BadReaderParamsError("The prototype request must be "
                                          "parseable as a LogReadRequest.")

    # Pass the parameters to logservice.fetch() to verify any underlying
    # constraints on types or values.  This only constructs an iterator, it
    # doesn't trigger any requests for actual log records.
    try:
      logservice.fetch(**params)
    except logservice.InvalidArgumentError, e:
      raise errors.BadReaderParamsError("One or more parameters are not valid "
                                        "inputs to logservice.fetch(): %s" % e)

Response: Validates the input reader s specification and all necessary parameters.


Instruction:
Given the following Python 3 function, write the documentation
def _next_file(self):
    while True:
      if self._bucket_iter:
        try:
          return self._bucket_iter.next().filename
        except StopIteration:
          self._bucket_iter = None
          self._bucket = None
      if self._index >= len(self._filenames):
        return
      filename = self._filenames[self._index]
      self._index += 1
      if self._delimiter is None or not filename.endswith(self._delimiter):
        return filename
      self._bucket = cloudstorage.listbucket(filename,
                                             delimiter=self._delimiter)
      self._bucket_iter = iter(self._bucket)

Response: Find next file in the list.


Instruction:
How would you explain what the following Python 3 function does
def validate(cls, mapper_spec):
    reader_spec = cls.get_params(mapper_spec, allow_old=False)

    # Bucket Name is required
    if cls.BUCKET_NAME_PARAM not in reader_spec:
      raise errors.BadReaderParamsError(
          "%s is required for Google Cloud Storage" %
          cls.BUCKET_NAME_PARAM)
    try:
      cloudstorage.validate_bucket_name(
          reader_spec[cls.BUCKET_NAME_PARAM])
    except ValueError, error:
      raise errors.BadReaderParamsError("Bad bucket name, %s" % (error))

    # Object Name(s) are required
    if cls.OBJECT_NAMES_PARAM not in reader_spec:
      raise errors.BadReaderParamsError(
          "%s is required for Google Cloud Storage" %
          cls.OBJECT_NAMES_PARAM)
    filenames = reader_spec[cls.OBJECT_NAMES_PARAM]
    if not isinstance(filenames, list):
      raise errors.BadReaderParamsError(
          "Object name list is not a list but a %s" %
          filenames.__class__.__name__)
    for filename in filenames:
      if not isinstance(filename, basestring):
        raise errors.BadReaderParamsError(
            "Object name is not a string but a %s" %
            filename.__class__.__name__)
    if cls.DELIMITER_PARAM in reader_spec:
      delimiter = reader_spec[cls.DELIMITER_PARAM]
      if not isinstance(delimiter, basestring):
        raise errors.BadReaderParamsError(
            "%s is not a string but a %s" %
            (cls.DELIMITER_PARAM, type(delimiter)))

Response: Validate the mapper specification.


Instruction:
How would you explain what the following Python 3 function does
def split_input(cls, mapper_spec):
    reader_spec = cls.get_params(mapper_spec, allow_old=False)
    bucket = reader_spec[cls.BUCKET_NAME_PARAM]
    filenames = reader_spec[cls.OBJECT_NAMES_PARAM]
    delimiter = reader_spec.get(cls.DELIMITER_PARAM)
    account_id = reader_spec.get(cls._ACCOUNT_ID_PARAM)
    buffer_size = reader_spec.get(cls.BUFFER_SIZE_PARAM)
    fail_on_missing_input = reader_spec.get(cls.FAIL_ON_MISSING_INPUT)

    # Gather the complete list of files (expanding wildcards)
    all_filenames = []
    for filename in filenames:
      if filename.endswith("*"):
        all_filenames.extend(
            [file_stat.filename for file_stat in cloudstorage.listbucket(
                "/" + bucket + "/" + filename[:-1], delimiter=delimiter,
                _account_id=account_id)])
      else:
        all_filenames.append("/%s/%s" % (bucket, filename))

    # Split into shards
    readers = []
    for shard in range(0, mapper_spec.shard_count):
      shard_filenames = all_filenames[shard::mapper_spec.shard_count]
      if shard_filenames:
        reader = cls(
            shard_filenames, buffer_size=buffer_size, _account_id=account_id,
            delimiter=delimiter)
        reader._fail_on_missing_input = fail_on_missing_input
        readers.append(reader)
    return readers

Response: Returns a list of InputReaders.


Instruction:
Can you implement a function in Python 3 that
returns the next input from this input reader a block of bytes.

Response:
def next(self):
    """Returns the next input from this input reader, a block of bytes.

    Non existent files will be logged and skipped. The file might have been
    removed after input splitting.

    Returns:
      The next input from this input reader in the form of a cloudstorage
      ReadBuffer that supports a File-like interface (read, readline, seek,
      tell, and close). An error may be raised if the file can not be opened.

    Raises:
      StopIteration: The list of files has been exhausted.
    """
    options = {}
    if self._buffer_size:
      options["read_buffer_size"] = self._buffer_size
    if self._account_id:
      options["_account_id"] = self._account_id
    while True:
      filename = self._next_file()
      if filename is None:
        raise StopIteration()
      try:
        start_time = time.time()
        handle = cloudstorage.open(filename, **options)

        ctx = context.get()
        if ctx:
          operation.counters.Increment(
              COUNTER_IO_READ_MSEC, int((time.time() - start_time) * 1000))(ctx)

        return handle
      except cloudstorage.NotFoundError:
        # Fail the job if we're strict on missing input.
        if getattr(self, "_fail_on_missing_input", False):
          raise errors.FailJobError(
              "File missing in GCS, aborting: %s" % filename)
        # Move on otherwise.
        logging.warning("File %s may have been removed. Skipping file.",
                        filename)


Instruction:
Here you have a function in Python 3, explain what it does
def next(self):
    while True:
      if not hasattr(self, "_cur_handle") or self._cur_handle is None:
        # If there are no more files, StopIteration is raised here
        self._cur_handle = super(_GoogleCloudStorageRecordInputReader,
                                 self).next()
      if not hasattr(self, "_record_reader") or self._record_reader is None:
        self._record_reader = records.RecordsReader(self._cur_handle)

      try:
        start_time = time.time()
        content = self._record_reader.read()

        ctx = context.get()
        if ctx:
          operation.counters.Increment(COUNTER_IO_READ_BYTES, len(content))(ctx)
          operation.counters.Increment(
              COUNTER_IO_READ_MSEC, int((time.time() - start_time) * 1000))(ctx)
        return content

      except EOFError:
        self._cur_handle = None
        self._record_reader = None

Response: Returns the next input from this input reader a record.


Instruction:
Create a Python 3 function to
return an input shard state for the remaining inputs.

Response:
def to_json(self):
    """Returns an input shard state for the remaining inputs.

    Returns:
      A json-izable version of the remaining InputReader.
    """
    result = super(_ReducerReader, self).to_json()
    result["current_key"] = self.encode_data(self.current_key)
    result["current_values"] = self.encode_data(self.current_values)
    return result


Instruction:
Can you generate the documentation for the following Python 3 function
def from_json(cls, json):
    result = super(_ReducerReader, cls).from_json(json)
    result.current_key = _ReducerReader.decode_data(json["current_key"])
    result.current_values = _ReducerReader.decode_data(json["current_values"])
    return result

Response: Creates an instance of the InputReader configured using the values of json.


Instruction:
Here you have a function in Python 3, explain what it does
def incr(self, counter_name, delta=1):
    self._state.counters_map.increment(counter_name, delta)

Response: Changes counter by delta.


Instruction:
Can you create a Python 3 function that
gets the current value of a counter.

Response:
def counter(self, counter_name, default=0):
    """Get the current counter value.

    Args:
      counter_name: name of the counter in string.
      default: default value in int if one doesn't exist.

    Returns:
      Current value of the counter.
    """
    return self._state.counters_map.get(counter_name, default)


Instruction:
Can you tell what is the following Python 3 function doing
def emit(self, value):
    if not self._tstate.output_writer:
      logging.error("emit is called, but no output writer is set.")
      return
    self._tstate.output_writer.write(value)

Response: Emits a value to the output writer.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def validate(cls, job_config):
    reader_params = job_config.input_reader_params

    # Bucket Name is required
    if cls.BUCKET_NAME_PARAM not in reader_params:
      raise errors.BadReaderParamsError(
          "%s is required for Google Cloud Storage" %
          cls.BUCKET_NAME_PARAM)
    try:
      cloudstorage.validate_bucket_name(
          reader_params[cls.BUCKET_NAME_PARAM])
    except ValueError, error:
      raise errors.BadReaderParamsError("Bad bucket name, %s" % (error))

    # Object Name(s) are required
    if cls.OBJECT_NAMES_PARAM not in reader_params:
      raise errors.BadReaderParamsError(
          "%s is required for Google Cloud Storage" %
          cls.OBJECT_NAMES_PARAM)
    filenames = reader_params[cls.OBJECT_NAMES_PARAM]
    if not isinstance(filenames, list):
      raise errors.BadReaderParamsError(
          "Object name list is not a list but a %s" %
          filenames.__class__.__name__)
    for filename in filenames:
      if not isinstance(filename, basestring):
        raise errors.BadReaderParamsError(
            "Object name is not a string but a %s" %
            filename.__class__.__name__)

    # Delimiter.
    if cls.DELIMITER_PARAM in reader_params:
      delimiter = reader_params[cls.DELIMITER_PARAM]
      if not isinstance(delimiter, basestring):
        raise errors.BadReaderParamsError(
            "%s is not a string but a %s" %
            (cls.DELIMITER_PARAM, type(delimiter)))

    # Buffer size.
    if cls.BUFFER_SIZE_PARAM in reader_params:
      buffer_size = reader_params[cls.BUFFER_SIZE_PARAM]
      if not isinstance(buffer_size, int):
        raise errors.BadReaderParamsError(
            "%s is not an int but a %s" %
            (cls.BUFFER_SIZE_PARAM, type(buffer_size)))

    # Path filter.
    if cls.PATH_FILTER_PARAM in reader_params:
      path_filter = reader_params[cls.PATH_FILTER_PARAM]
      if not isinstance(path_filter, PathFilter):
        raise errors.BadReaderParamsError(
            "%s is not an instance of PathFilter but %s." %
            (cls.PATH_FILTER_PARAM, type(path_filter)))

Response: Validate the mapper specification.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def split_input(cls, job_config):
    reader_params = job_config.input_reader_params
    bucket = reader_params[cls.BUCKET_NAME_PARAM]
    filenames = reader_params[cls.OBJECT_NAMES_PARAM]
    delimiter = reader_params.get(cls.DELIMITER_PARAM)
    account_id = reader_params.get(cls._ACCOUNT_ID_PARAM)
    buffer_size = reader_params.get(cls.BUFFER_SIZE_PARAM)
    path_filter = reader_params.get(cls.PATH_FILTER_PARAM)

    # Gather the complete list of files (expanding wildcards)
    all_filenames = []
    for filename in filenames:
      if filename.endswith("*"):
        all_filenames.extend(
            [file_stat.filename for file_stat in cloudstorage.listbucket(
                "/" + bucket + "/" + filename[:-1], delimiter=delimiter,
                _account_id=account_id)])
      else:
        all_filenames.append("/%s/%s" % (bucket, filename))

    # Split into shards
    readers = []
    for shard in range(0, job_config.shard_count):
      shard_filenames = all_filenames[shard::job_config.shard_count]
      if shard_filenames:
        readers.append(cls(
            shard_filenames, buffer_size=buffer_size, _account_id=account_id,
            delimiter=delimiter, path_filter=path_filter))
    return readers

Response: Splits the input files into shards and returns a list of InputReaders.


Instruction:
Implement a Python 3 function for
returning a handler to the next file in the list.

Response:
def next(self):
    """Returns a handler to the next file.

    Non existent files will be logged and skipped. The file might have been
    removed after input splitting.

    Returns:
      The next input from this input reader in the form of a cloudstorage
      ReadBuffer that supports a File-like interface (read, readline, seek,
      tell, and close). An error may be raised if the file can not be opened.

    Raises:
      StopIteration: The list of files has been exhausted.
    """
    options = {}
    if self._buffer_size:
      options["read_buffer_size"] = self._buffer_size
    if self._account_id:
      options["_account_id"] = self._account_id
    while True:
      filename = self._next_file()
      if filename is None:
        raise StopIteration()
      if (self._path_filter and
          not self._path_filter.accept(self._slice_ctx, filename)):
        continue
      try:
        start_time = time.time()
        handle = cloudstorage.open(filename, **options)
        self._slice_ctx.incr(self.COUNTER_IO_READ_MSEC,
                             int(time.time() - start_time) * 1000)
        self._slice_ctx.incr(self.COUNTER_FILE_READ)
        return handle
      except cloudstorage.NotFoundError:
        logging.warning("File %s may have been removed. Skipping file.",
                        filename)
        self._slice_ctx.incr(self.COUNTER_FILE_MISSING)


Instruction:
Can you implement a function in Python 3 that
returns the next input from this input reader a record.

Response:
def next(self):
    """Returns the next input from this input reader, a record.

    Returns:
      The next input from this input reader in the form of a record read from
      an LevelDB file.

    Raises:
      StopIteration: The ordered set records has been exhausted.
    """
    while True:
      if not hasattr(self, "_cur_handle") or self._cur_handle is None:
        # If there are no more files, StopIteration is raised here
        self._cur_handle = super(GCSRecordInputReader, self).next()
      if not hasattr(self, "_record_reader") or self._record_reader is None:
        self._record_reader = records.RecordsReader(self._cur_handle)

      try:
        start_time = time.time()
        content = self._record_reader.read()
        self._slice_ctx.incr(self.COUNTER_IO_READ_BYTE, len(content))
        self._slice_ctx.incr(self.COUNTER_IO_READ_MSEC,
                             int(time.time() - start_time) * 1000)
        return content
      except EOFError:
        self._cur_handle = None
        self._record_reader = None


Instruction:
Given the following Python 3 function, write the documentation
def _get_query_spec(cls, params):
    entity_kind = params[cls.ENTITY_KIND_PARAM]
    filters = params.get(cls.FILTERS_PARAM)
    app = params.get(cls._APP_PARAM)
    ns = params.get(cls.NAMESPACE_PARAM)

    return model.QuerySpec(
        entity_kind=cls._get_raw_entity_kind(entity_kind),
        keys_only=bool(params.get(cls.KEYS_ONLY_PARAM, False)),
        filters=filters,
        batch_size=int(params.get(cls.BATCH_SIZE_PARAM, cls._BATCH_SIZE)),
        model_class_path=entity_kind,
        app=app,
        ns=ns)

Response: Construct a model. QuerySpec from model. MapperSpec.


Instruction:
Create a Python 3 function to
get a list of key_ranges.KeyRanges objects, one for each shard. This method uses scatter index to split each namespace into pieces and assign those pieces to shards. Args: app: app_id in str. namespaces: a list of namespaces in str. shard_count: number of shards to split. query_spec: model.QuerySpec. Returns: a list of key_ranges.KeyRanges objects.

Response:
def _to_key_ranges_by_shard(cls, app, namespaces, shard_count, query_spec):
    """Get a list of key_ranges.KeyRanges objects, one for each shard.

    This method uses scatter index to split each namespace into pieces
    and assign those pieces to shards.

    Args:
      app: app_id in str.
      namespaces: a list of namespaces in str.
      shard_count: number of shards to split.
      query_spec: model.QuerySpec.

    Returns:
      a list of key_ranges.KeyRanges objects.
    """
    key_ranges_by_ns = []
    # Split each ns into n splits. If a ns doesn't have enough scatter to
    # split into n, the last few splits are None.
    for namespace in namespaces:
      ranges = cls._split_ns_by_scatter(
          shard_count,
          namespace,
          query_spec.entity_kind,
          app)
      # The nth split of each ns will be assigned to the nth shard.
      # Shuffle so that None are not all by the end.
      random.shuffle(ranges)
      key_ranges_by_ns.append(ranges)

    # KeyRanges from different namespaces might be very different in size.
    # Use round robin to make sure each shard can have at most one split
    # or a None from a ns.
    ranges_by_shard = [[] for _ in range(shard_count)]
    for ranges in key_ranges_by_ns:
      for i, k_range in enumerate(ranges):
        if k_range:
          ranges_by_shard[i].append(k_range)

    key_ranges_by_shard = []
    for ranges in ranges_by_shard:
      if ranges:
        key_ranges_by_shard.append(key_ranges.KeyRangesFactory.create_from_list(
            ranges))
    return key_ranges_by_shard


Instruction:
Write a Python 3 function that can
split a namespace by a number of shards.

Response:
def _split_ns_by_scatter(cls,
                           shard_count,
                           namespace,
                           raw_entity_kind,
                           app):
    """Split a namespace by scatter index into key_range.KeyRange.

    TODO(user): Power this with key_range.KeyRange.compute_split_points.

    Args:
      shard_count: number of shards.
      namespace: namespace name to split. str.
      raw_entity_kind: low level datastore API entity kind.
      app: app id in str.

    Returns:
      A list of key_range.KeyRange objects. If there are not enough entities to
    splits into requested shards, the returned list will contain KeyRanges
    ordered lexicographically with any Nones appearing at the end.
    """
    if shard_count == 1:
      # With one shard we don't need to calculate any split points at all.
      return [key_range.KeyRange(namespace=namespace, _app=app)]

    ds_query = datastore.Query(kind=raw_entity_kind,
                               namespace=namespace,
                               _app=app,
                               keys_only=True)
    ds_query.Order("__scatter__")
    oversampling_factor = 32
    random_keys = ds_query.Get(shard_count * oversampling_factor)

    if not random_keys:
      # There are no entities with scatter property. We have no idea
      # how to split.
      return ([key_range.KeyRange(namespace=namespace, _app=app)] +
              [None] * (shard_count - 1))

    random_keys.sort()

    if len(random_keys) >= shard_count:
      # We've got a lot of scatter values. Sample them down.
      random_keys = cls._choose_split_points(random_keys, shard_count)

    k_ranges = []

    k_ranges.append(key_range.KeyRange(
        key_start=None,
        key_end=random_keys[0],
        direction=key_range.KeyRange.ASC,
        include_start=False,
        include_end=False,
        namespace=namespace,
        _app=app))

    for i in range(0, len(random_keys) - 1):
      k_ranges.append(key_range.KeyRange(
          key_start=random_keys[i],
          key_end=random_keys[i+1],
          direction=key_range.KeyRange.ASC,
          include_start=True,
          include_end=False,
          namespace=namespace,
          _app=app))

    k_ranges.append(key_range.KeyRange(
        key_start=random_keys[-1],
        key_end=None,
        direction=key_range.KeyRange.ASC,
        include_start=True,
        include_end=False,
        namespace=namespace,
        _app=app))

    if len(k_ranges) < shard_count:
      # We need to have as many shards as it was requested. Add some Nones.
      k_ranges += [None] * (shard_count - len(k_ranges))
    return k_ranges


Instruction:
Can you generate the documentation for the following Python 3 function
def _setup_constants(alphabet=NAMESPACE_CHARACTERS,
                     max_length=MAX_NAMESPACE_LENGTH,
                     batch_size=NAMESPACE_BATCH_SIZE):
  """Calculate derived constant values. Only useful for testing."""

  global NAMESPACE_CHARACTERS
  global MAX_NAMESPACE_LENGTH
  # pylint: disable=global-variable-undefined
  global MAX_NAMESPACE
  global _LEX_DISTANCE
  global NAMESPACE_BATCH_SIZE

  NAMESPACE_CHARACTERS = alphabet
  MAX_NAMESPACE_LENGTH = max_length
  MAX_NAMESPACE = NAMESPACE_CHARACTERS[-1] * MAX_NAMESPACE_LENGTH
  NAMESPACE_BATCH_SIZE = batch_size

  # _LEX_DISTANCE will contain the lexical distance between two adjacent
  # characters in NAMESPACE_CHARACTERS at each character index. This is used
  # to calculate the ordinal for each string. Example:
  # NAMESPACE_CHARACTERS = 'ab'
  # MAX_NAMESPACE_LENGTH = 3
  # _LEX_DISTANCE = [1, 3, 7]
  # ''    => 0
  # 'a'   => 1
  # 'aa'  => 2
  # 'aaa' => 3
  # 'aab' => 4 - Distance between 'aaa' and 'aab' is 1.
  # 'ab'  => 5 - Distance between 'aa' and 'ab' is 3.
  # 'aba' => 6
  # 'abb' => 7
  # 'b'   => 8 - Distance between 'a' and 'b' is 7.
  # 'ba'  => 9
  # 'baa' => 10
  # 'bab' => 11
  # ...
  # _namespace_to_ord('bab') = (1 * 7 + 1) + (0 * 3 + 1) + (1 * 1 + 1) = 11
  _LEX_DISTANCE = [1]
  for i in range(1, MAX_NAMESPACE_LENGTH):
    _LEX_DISTANCE.append(
        _LEX_DISTANCE[i-1] * len(NAMESPACE_CHARACTERS) + 1)
  # pylint: disable=undefined-loop-variable
  del i

Response: Calculate derived constant values. Only useful for testing.


Instruction:
Given the following Python 3 function, write the documentation
def _ord_to_namespace(n, _max_length=None):
  if _max_length is None:
    _max_length = MAX_NAMESPACE_LENGTH

  length = _LEX_DISTANCE[_max_length - 1]
  if n == 0:
    return ''
  n -= 1
  return (NAMESPACE_CHARACTERS[n / length] +
          _ord_to_namespace(n % length, _max_length - 1))

Response: Convert a namespace ordinal to a namespace string.


Instruction:
Can you tell what is the following Python 3 function doing
def _namespace_to_ord(namespace):
  n = 0
  for i, c in enumerate(namespace):
    n += (_LEX_DISTANCE[MAX_NAMESPACE_LENGTH - i- 1] *
          NAMESPACE_CHARACTERS.index(c)
          + 1)
  return n

Response: Converts a namespace string into an int representing its lexographic order.


Instruction:
Can you create a Python 3 function that
returns the key for a namespace.

Response:
def _key_for_namespace(namespace, app):
  """Return the __namespace__ key for a namespace.

  Args:
    namespace: The namespace whose key is requested.
    app: The id of the application that the key belongs to.

  Returns:
    A db.Key representing the namespace.
  """
  if namespace:
    return db.Key.from_path(metadata.Namespace.KIND_NAME,
                            namespace,
                            _app=app)
  else:
    return db.Key.from_path(metadata.Namespace.KIND_NAME,
                            metadata.Namespace.EMPTY_NAMESPACE_ID,
                            _app=app)


Instruction:
How would you explain what the following Python 3 function does
def split_range(self):
    if self.is_single_namespace:
      return [self]

    mid_point = (_namespace_to_ord(self.namespace_start) +
                 _namespace_to_ord(self.namespace_end)) // 2

    return [NamespaceRange(self.namespace_start,
                           _ord_to_namespace(mid_point),
                           _app=self.app),
            NamespaceRange(_ord_to_namespace(mid_point+1),
                           self.namespace_end,
                           _app=self.app)]

Response: Splits the NamespaceRange into two nearly equal - sized ranges.


Instruction:
Write a Python 3 script to
return a copy of this NamespaceName with a new namespace_start.

Response:
def with_start_after(self, after_namespace):
    """Returns a copy of this NamespaceName with a new namespace_start.

    Args:
      after_namespace: A namespace string.

    Returns:
      A NamespaceRange object whose namespace_start is the lexographically next
      namespace after the given namespace string.

    Raises:
      ValueError: if the NamespaceRange includes only a single namespace.
    """
    namespace_start = _ord_to_namespace(_namespace_to_ord(after_namespace) + 1)
    return NamespaceRange(namespace_start, self.namespace_end, _app=self.app)


Instruction:
How would you implement a function in Python 3 that
returns a datastore. Query that generates all namespaces in the range.

Response:
def make_datastore_query(self, cursor=None):
    """Returns a datastore.Query that generates all namespaces in the range.

    Args:
      cursor: start cursor for the query.

    Returns:
      A datastore.Query instance that generates db.Keys for each namespace in
      the NamespaceRange.
    """
    filters = {}
    filters['__key__ >= '] = _key_for_namespace(
        self.namespace_start, self.app)
    filters['__key__ <= '] = _key_for_namespace(
        self.namespace_end, self.app)

    return datastore.Query('__namespace__',
                           filters=filters,
                           keys_only=True,
                           cursor=cursor,
                           _app=self.app)


Instruction:
Write a Python 3 function for
returning a new NamespaceRange with leading non - existant namespaces removed.

Response:
def normalized_start(self):
    """Returns a NamespaceRange with leading non-existant namespaces removed.

    Returns:
      A copy of this NamespaceRange whose namespace_start is adjusted to exclude
      the portion of the range that contains no actual namespaces in the
      datastore. None is returned if the NamespaceRange contains no actual
      namespaces in the datastore.
    """
    namespaces_after_key = list(self.make_datastore_query().Run(limit=1))

    if not namespaces_after_key:
      return None

    namespace_after_key = namespaces_after_key[0].name() or ''
    return NamespaceRange(namespace_after_key,
                          self.namespace_end,
                          _app=self.app)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def to_json_object(self):
    obj_dict = dict(namespace_start=self.namespace_start,
                    namespace_end=self.namespace_end)
    if self.app is not None:
      obj_dict['app'] = self.app
    return obj_dict

Response: Returns a dict representation that can be serialized to JSON.


Instruction:
Create a Python 3 function for
splitting the complete NamespaceRange into n equally - sized NamespaceRanges.

Response:
def split(cls,
            n,
            contiguous,
            can_query=itertools.chain(itertools.repeat(True, 50),
                                      itertools.repeat(False)).next,
            _app=None):
    # pylint: disable=g-doc-args
    """Splits the complete NamespaceRange into n equally-sized NamespaceRanges.

    Args:
      n: The maximum number of NamespaceRanges to return. Fewer than n
          namespaces may be returned.
      contiguous: If True then the returned NamespaceRanges will cover the
          entire space of possible namespaces (i.e. from MIN_NAMESPACE to
          MAX_NAMESPACE) without gaps. If False then the returned
          NamespaceRanges may exclude namespaces that don't appear in the
          datastore.
      can_query: A function that returns True if split() can query the datastore
          to generate more fair namespace range splits, and False otherwise.
          If not set then split() is allowed to make 50 datastore queries.

    Returns:
      A list of at most n NamespaceRanges representing a near-equal distribution
      of actual existant datastore namespaces. The returned list will be sorted
      lexographically.

    Raises:
      ValueError: if n is < 1.
    """
    if n < 1:
      raise ValueError('n must be >= 1')

    ranges = None
    if can_query():
      if not contiguous:
        ns_keys = get_namespace_keys(_app, n + 1)
        if not ns_keys:
          return []
        else:
          if len(ns_keys) <= n:
            # If you have less actual namespaces than number of NamespaceRanges
            # to return, then just return the list of those namespaces.
            ns_range = []
            for ns_key in ns_keys:
              ns_range.append(NamespaceRange(ns_key.name() or '',
                                             ns_key.name() or '',
                                             _app=_app))
            return sorted(ns_range,
                          key=lambda ns_range: ns_range.namespace_start)
          # Use the first key and save the initial normalized_start() call.
          ranges = [NamespaceRange(ns_keys[0].name() or '', _app=_app)]
      else:
        ns_range = NamespaceRange(_app=_app).normalized_start()
        if ns_range is None:
          return [NamespaceRange(_app=_app)]
        ranges = [ns_range]
    else:
      ranges = [NamespaceRange(_app=_app)]

    singles = []
    while ranges and (len(ranges) + len(singles)) < n:
      namespace_range = ranges.pop(0)
      if namespace_range.is_single_namespace:
        singles.append(namespace_range)
      else:
        left, right = namespace_range.split_range()
        if can_query():
          right = right.normalized_start()
        if right is not None:
          ranges.append(right)
        ranges.append(left)
    ns_ranges = sorted(singles + ranges,
                       key=lambda ns_range: ns_range.namespace_start)

    if contiguous:
      if not ns_ranges:
        # This condition is possible if every namespace was deleted after the
        # first call to ns_range.normalized_start().
        return [NamespaceRange(_app=_app)]

      continuous_ns_ranges = []
      for i in range(len(ns_ranges)):
        if i == 0:
          namespace_start = MIN_NAMESPACE
        else:
          namespace_start = ns_ranges[i].namespace_start

        if i == len(ns_ranges) - 1:
          namespace_end = MAX_NAMESPACE
        else:
          namespace_end = _ord_to_namespace(
              _namespace_to_ord(ns_ranges[i+1].namespace_start) - 1)

        continuous_ns_ranges.append(NamespaceRange(namespace_start,
                                                   namespace_end,
                                                   _app=_app))
      return continuous_ns_ranges
    else:
      return ns_ranges


Instruction:
Given the following Python 3 function, write the documentation
def append(self, data):
    data_length = len(data)
    if self._size + data_length > self._flush_size:
      self.flush()

    if not self._exclusive and data_length > _FILE_POOL_MAX_SIZE:
      raise errors.Error(
          "Too big input %s (%s)."  % (data_length, _FILE_POOL_MAX_SIZE))
    else:
      self._buffer.append(data)
      self._size += data_length

    if self._size > self._flush_size:
      self.flush()

Response: Append data to a file.


Instruction:
Here you have a function in Python 3, explain what it does
def flush(self):
    # Write data to in-memory buffer first.
    buf = cStringIO.StringIO()
    with records.RecordsWriter(buf) as w:
      for record in self._buffer:
        w.write(record)
      w._pad_block()
    str_buf = buf.getvalue()
    buf.close()

    if not self._exclusive and len(str_buf) > _FILE_POOL_MAX_SIZE:
      # Shouldn't really happen because of flush size.
      raise errors.Error(
          "Buffer too big. Can't write more than %s bytes in one request: "
          "risk of writes interleaving. Got: %s" %
          (_FILE_POOL_MAX_SIZE, len(str_buf)))

    # Write data to file.
    start_time = time.time()
    self._write(str_buf)
    if self._ctx:
      operation.counters.Increment(
          COUNTER_IO_WRITE_BYTES, len(str_buf))(self._ctx)
      operation.counters.Increment(
          COUNTER_IO_WRITE_MSEC,
          int((time.time() - start_time) * 1000))(self._ctx)

    # reset buffer
    self._buffer = []
    self._size = 0
    gc.collect()

Response: Flushes the contents of the pool to the file.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _write(self, str_buf):
    self._filehandle.write(str_buf)
    self._buf_size += len(str_buf)

Response: Writes a string to GCS.


Instruction:
How would you implement a function in Python 3 that
flushes the contents of the GCS record pool.

Response:
def flush(self, force=False):
    """Flush pool contents.

    Args:
      force: Inserts additional padding to achieve the minimum block size
        required for GCS.
    """
    super(GCSRecordsPool, self).flush()
    if force:
      extra_padding = self._buf_size % self._GCS_BLOCK_SIZE
      if extra_padding > 0:
        self._write("\x00" * (self._GCS_BLOCK_SIZE - extra_padding))
    self._filehandle.flush()


Instruction:
How would you implement a function in Python 3 that
returns bucket used for writing tmp files.

Response:
def _get_tmp_gcs_bucket(cls, writer_spec):
    """Returns bucket used for writing tmp files."""
    if cls.TMP_BUCKET_NAME_PARAM in writer_spec:
      return writer_spec[cls.TMP_BUCKET_NAME_PARAM]
    return cls._get_gcs_bucket(writer_spec)


Instruction:
Here you have a function in Python 3, explain what it does
def _get_tmp_account_id(cls, writer_spec):
    # pick tmp id iff tmp bucket is set explicitly
    if cls.TMP_BUCKET_NAME_PARAM in writer_spec:
      return writer_spec.get(cls._TMP_ACCOUNT_ID_PARAM, None)
    return cls._get_account_id(writer_spec)

Response: Returns the account id to use with tmp bucket.


Instruction:
How would you implement a function in Python 3 that
generates a filename for a particular output.

Response:
def _generate_filename(cls, writer_spec, name, job_id, num,
                         attempt=None, seg_index=None):
    """Generates a filename for a particular output.

    Args:
      writer_spec: specification dictionary for the output writer.
      name: name of the job.
      job_id: the ID number assigned to the job.
      num: shard number.
      attempt: the shard attempt number.
      seg_index: index of the seg. None means the final output.

    Returns:
      a string containing the filename.

    Raises:
      BadWriterParamsError: if the template contains any errors such as invalid
        syntax or contains unknown substitution placeholders.
    """
    naming_format = cls._TMP_FILE_NAMING_FORMAT
    if seg_index is None:
      naming_format = writer_spec.get(cls.NAMING_FORMAT_PARAM,
                                      cls._DEFAULT_NAMING_FORMAT)

    template = string.Template(naming_format)
    try:
      # Check that template doesn't use undefined mappings and is formatted well
      if seg_index is None:
        return template.substitute(name=name, id=job_id, num=num)
      else:
        return template.substitute(name=name, id=job_id, num=num,
                                   attempt=attempt,
                                   seg=seg_index)
    except ValueError, error:
      raise errors.BadWriterParamsError("Naming template is bad, %s" % (error))
    except KeyError, error:
      raise errors.BadWriterParamsError("Naming template '%s' has extra "
                                        "mappings, %s" % (naming_format, error))


Instruction:
Can you tell what is the following Python 3 function doing
def validate(cls, mapper_spec):
    writer_spec = cls.get_params(mapper_spec, allow_old=False)

    # Bucket Name is required
    if cls.BUCKET_NAME_PARAM not in writer_spec:
      raise errors.BadWriterParamsError(
          "%s is required for Google Cloud Storage" %
          cls.BUCKET_NAME_PARAM)
    try:
      cloudstorage.validate_bucket_name(
          writer_spec[cls.BUCKET_NAME_PARAM])
    except ValueError, error:
      raise errors.BadWriterParamsError("Bad bucket name, %s" % (error))

    # Validate the naming format does not throw any errors using dummy values
    cls._generate_filename(writer_spec, "name", "id", 0)
    cls._generate_filename(writer_spec, "name", "id", 0, 1, 0)

Response: Validate the mapper specification.


Instruction:
Given the following Python 3 function, write the documentation
def _open_file(cls, writer_spec, filename_suffix, use_tmp_bucket=False):
    if use_tmp_bucket:
      bucket = cls._get_tmp_gcs_bucket(writer_spec)
      account_id = cls._get_tmp_account_id(writer_spec)
    else:
      bucket = cls._get_gcs_bucket(writer_spec)
      account_id = cls._get_account_id(writer_spec)

    # GoogleCloudStorage format for filenames, Initial slash is required
    filename = "/%s/%s" % (bucket, filename_suffix)

    content_type = writer_spec.get(cls.CONTENT_TYPE_PARAM, None)

    options = {}
    if cls.ACL_PARAM in writer_spec:
      options["x-goog-acl"] = writer_spec.get(cls.ACL_PARAM)

    return cloudstorage.open(filename, mode="w", content_type=content_type,
                             options=options, _account_id=account_id)

Response: Opens a gcs file for writing.


Instruction:
Can you create a Python 3 function that
writes data to the Google Cloud Storage file.

Response:
def write(self, data):
    """Write data to the GoogleCloudStorage file.

    Args:
      data: string containing the data to be written.
    """
    start_time = time.time()
    self._get_write_buffer().write(data)
    ctx = context.get()
    operation.counters.Increment(COUNTER_IO_WRITE_BYTES, len(data))(ctx)
    operation.counters.Increment(
        COUNTER_IO_WRITE_MSEC, int((time.time() - start_time) * 1000))(ctx)


Instruction:
Here you have a function in Python 3, explain what it does
def _create(cls, writer_spec, filename_suffix):
    writer = cls._open_file(writer_spec, filename_suffix)
    return cls(writer, writer_spec=writer_spec)

Response: Helper method that actually creates the file in cloud storage.


Instruction:
Can you tell what is the following Python 3 function doing
def _rewrite_tmpfile(self, mainfile, tmpfile, writer_spec):
    if mainfile.closed:
      # can happen when finalize fails
      return

    account_id = self._get_tmp_account_id(writer_spec)
    f = cloudstorage_api.open(tmpfile, _account_id=account_id)
    # both reads and writes are buffered - the number here doesn't matter
    data = f.read(self._REWRITE_BLOCK_SIZE)
    while data:
      mainfile.write(data)
      data = f.read(self._REWRITE_BLOCK_SIZE)
    f.close()
    mainfile.flush()

Response: Copies contents of tmpfile to mainfile.


Instruction:
Create a Python 3 function to
create a new random - named tmpfile.

Response:
def _create_tmpfile(cls, status):
    """Creates a new random-named tmpfile."""

    # We can't put the tmpfile in the same directory as the output. There are
    # rare circumstances when we leave trash behind and we don't want this trash
    # to be loaded into bigquery and/or used for restore.
    #
    # We used mapreduce id, shard number and attempt and 128 random bits to make
    # collisions virtually impossible.
    tmpl = string.Template(cls._TMPFILE_PATTERN)
    filename = tmpl.substitute(
        id=status.mapreduce_id, shard=status.shard,
        random=random.getrandbits(cls._RAND_BITS))

    return cls._open_file(status.writer_spec, filename, use_tmp_bucket=True)


Instruction:
Can you generate the documentation for the following Python 3 function
def _try_to_clean_garbage(self, writer_spec, exclude_list=()):
    # Try to remove garbage (if any). Note that listbucket is not strongly
    # consistent so something might survive.
    tmpl = string.Template(self._TMPFILE_PREFIX)
    prefix = tmpl.substitute(
        id=self.status.mapreduce_id, shard=self.status.shard)
    bucket = self._get_tmp_gcs_bucket(writer_spec)
    account_id = self._get_tmp_account_id(writer_spec)
    for f in cloudstorage.listbucket("/%s/%s" % (bucket, prefix),
                                     _account_id=account_id):
      if f.filename not in exclude_list:
        self._remove_tmpfile(f.filename, self.status.writer_spec)

Response: Try to remove any files created by this shard that aren t needed.


Instruction:
Write a Python 3 script to
get weights for each offset in str of certain max length.

Response:
def _get_weights(max_length):
  """Get weights for each offset in str of certain max length.

  Args:
    max_length: max length of the strings.

  Returns:
    A list of ints as weights.

  Example:
    If max_length is 2 and alphabet is "ab", then we have order "", "a", "aa",
  "ab", "b", "ba", "bb". So the weight for the first char is 3.
  """
  weights = [1]
  for i in range(1, max_length):
    weights.append(weights[i-1] * len(_ALPHABET) + 1)
  weights.reverse()
  return weights


Instruction:
Write a Python 3 function for
converting a string to its lexicographical order.

Response:
def _str_to_ord(content, weights):
  """Converts a string to its lexicographical order.

  Args:
    content: the string to convert. Of type str.
    weights: weights from _get_weights.

  Returns:
    an int or long that represents the order of this string. "" has order 0.
  """
  ordinal = 0
  for i, c in enumerate(content):
    ordinal += weights[i] * _ALPHABET.index(c) + 1
  return ordinal


Instruction:
Write a Python 3 function that can
reverse function of _str_to_ord.

Response:
def _ord_to_str(ordinal, weights):
  """Reverse function of _str_to_ord."""
  chars = []
  for weight in weights:
    if ordinal == 0:
      return "".join(chars)
    ordinal -= 1
    index, ordinal = divmod(ordinal, weight)
    chars.append(_ALPHABET[index])
  return "".join(chars)


Instruction:
Given the following Python 3 function, write the documentation
def _get_range_from_filters(cls, filters, model_class):
    if not filters:
      return None, None, None

    range_property = None
    start_val = None
    end_val = None
    start_filter = None
    end_filter = None
    for f in filters:
      prop, op, val = f

      if op in [">", ">=", "<", "<="]:
        if range_property and range_property != prop:
          raise errors.BadReaderParamsError(
              "Range on only one property is supported.")
        range_property = prop

        if val is None:
          raise errors.BadReaderParamsError(
              "Range can't be None in filter %s", f)

        if op in [">", ">="]:
          if start_val is not None:
            raise errors.BadReaderParamsError(
                "Operation %s is specified more than once.", op)
          start_val = val
          start_filter = f
        else:
          if end_val is not None:
            raise errors.BadReaderParamsError(
                "Operation %s is specified more than once.", op)
          end_val = val
          end_filter = f
      elif op != "=":
        raise errors.BadReaderParamsError(
            "Only < <= > >= = are supported as operation. Got %s", op)

    if not range_property:
      return None, None, None

    if start_val is None or end_val is None:
      raise errors.BadReaderParamsError(
          "Filter should contains a complete range on property %s",
          range_property)
    if issubclass(model_class, db.Model):
      property_obj = model_class.properties()[range_property]
    else:
      property_obj = (
          model_class._properties[  # pylint: disable=protected-access
              range_property])
    supported_properties = (
        _DISCRETE_PROPERTY_SPLIT_FUNCTIONS.keys() +
        _CONTINUOUS_PROPERTY_SPLIT_FUNCTIONS.keys())
    if not isinstance(property_obj, tuple(supported_properties)):
      raise errors.BadReaderParamsError(
          "Filtered property %s is not supported by sharding.", range_property)
    if not start_val < end_val:
      raise errors.BadReaderParamsError(
          "Start value %s should be smaller than end value %s",
          start_val, end_val)

    return property_obj, start_filter, end_filter

Response: This method returns the property range from the user supplied filters.


Instruction:
Can you tell what is the following Python 3 function doing
def split(self, n):
    new_range_filters = []
    name = self.start[0]
    prop_cls = self.prop.__class__
    if prop_cls in _DISCRETE_PROPERTY_SPLIT_FUNCTIONS:
      splitpoints = _DISCRETE_PROPERTY_SPLIT_FUNCTIONS[prop_cls](
          self.start[2], self.end[2], n,
          self.start[1] == ">=", self.end[1] == "<=")
      start_filter = (name, ">=", splitpoints[0])
      for p in splitpoints[1:]:
        end_filter = (name, "<", p)
        new_range_filters.append([start_filter, end_filter])
        start_filter = (name, ">=", p)
    else:
      splitpoints = _CONTINUOUS_PROPERTY_SPLIT_FUNCTIONS[prop_cls](
          self.start[2], self.end[2], n)
      start_filter = self.start
      for p in splitpoints:
        end_filter = (name, "<", p)
        new_range_filters.append([start_filter, end_filter])
        start_filter = (name, ">=", p)
      new_range_filters.append([start_filter, self.end])

    for f in new_range_filters:
      f.extend(self._equality_filters)

    return [self.__class__(f, self.model_class_path) for f in new_range_filters]

Response: Evenly split this range into contiguous non overlapping subranges.


Instruction:
Write a Python 3 function that can
make a query of entities within this range.

Response:
def make_query(self, ns):
    """Make a query of entities within this range.

    Query options are not supported. They should be specified when the query
    is run.

    Args:
      ns: namespace of this query.

    Returns:
      a db.Query or ndb.Query, depends on the model class's type.
    """
    if issubclass(self.model_class, db.Model):
      query = db.Query(self.model_class, namespace=ns)
      for f in self.filters:
        query.filter("%s %s" % (f[0], f[1]), f[2])
    else:
      query = self.model_class.query(namespace=ns)
      for f in self.filters:
        query = query.filter(ndb.FilterNode(*f))
    return query


Instruction:
Can you generate the documentation for the following Python 3 function
def validate(cls, job_config):
    if job_config.output_writer_cls != cls:
      raise errors.BadWriterParamsError(
          "Expect output writer class %r, got %r." %
          (cls, job_config.output_writer_cls))

Response:
Validates relevant parameters.

    This method can validate fields which it deems relevant.

    Args:
      job_config: an instance of map_job.JobConfig.

    Raises:
      errors.BadWriterParamsError: required parameters are missing or invalid.


Instruction:
Write a Python 3 script for
committing the output references when a shard finishes.

Response:
def commit_output(cls, shard_ctx, iterator):
    """Saves output references when a shard finishes.

    Inside end_shard(), an output writer can optionally use this method
    to persist some references to the outputs from this shard
    (e.g a list of filenames)

    Args:
      shard_ctx: map_job_context.ShardContext for this shard.
      iterator: an iterator that yields json serializable
        references to the outputs from this shard.
        Contents from the iterator can be accessible later via
        map_job.Job.get_outputs.
    """
    # We accept an iterator just in case output references get too big.
    outs = tuple(iterator)
    shard_ctx._state.writer_state["outs"] = outs


Instruction:
Implement a Python 3 function for
creating a new handlers map.

Response:
def create_handlers_map():
  """Create new handlers map.

  Returns:
    list of (regexp, handler) pairs for WSGIApplication constructor.
  """
  pipeline_handlers_map = []

  if pipeline:
    pipeline_handlers_map = pipeline.create_handlers_map(prefix=".*/pipeline")

  return pipeline_handlers_map + [
      # Task queue handlers.
      # Always suffix by mapreduce_id or shard_id for log analysis purposes.
      # mapreduce_id or shard_id also presents in headers or payload.
      (r".*/worker_callback.*", handlers.MapperWorkerCallbackHandler),
      (r".*/controller_callback.*", handlers.ControllerCallbackHandler),
      (r".*/kickoffjob_callback.*", handlers.KickOffJobHandler),
      (r".*/finalizejob_callback.*", handlers.FinalizeJobHandler),

      # RPC requests with JSON responses
      # All JSON handlers should have /command/ prefix.
      (r".*/command/start_job", handlers.StartJobHandler),
      (r".*/command/cleanup_job", handlers.CleanUpJobHandler),
      (r".*/command/abort_job", handlers.AbortJobHandler),
      (r".*/command/list_configs", status.ListConfigsHandler),
      (r".*/command/list_jobs", status.ListJobsHandler),
      (r".*/command/get_job_detail", status.GetJobDetailHandler),

      # UI static files
      (STATIC_RE, status.ResourceHandler),

      # Redirect non-file URLs that do not end in status/detail to status page.
      (r".*", RedirectHandler),
      ]


Instruction:
Can you generate the documentation for the following Python 3 function
def from_json(cls, json):
    if json["name"] in _KEYRANGES_CLASSES:
      return _KEYRANGES_CLASSES[json["name"]].from_json(json)
    raise ValueError("Invalid json %s", json)

Response:
Deserialize from json.

    Args:
      json: a dict of json compatible fields.

    Returns:
      a KeyRanges object.

    Raises:
      ValueError: if the json is invalid.


Instruction:
Write a Python 3 script for
splitting text into list of sentences.

Response:
def split_into_sentences(s):
  """Split text into list of sentences."""
  s = re.sub(r"\s+", " ", s)
  s = re.sub(r"[\\.\\?\\!]", "\n", s)
  return s.split("\n")


Instruction:
Can you create a Python 3 function that
splits a sentence into list of words.

Response:
def split_into_words(s):
  """Split a sentence into list of words."""
  s = re.sub(r"\W+", " ", s)
  s = re.sub(r"[_0-9]+", " ", s)
  return s.split()


Instruction:
Write a Python 3 script for
indexing demo map function.

Response:
def index_map(data):
  """Index demo map function."""
  (entry, text_fn) = data
  text = text_fn()

  logging.debug("Got %s", entry.filename)
  for s in split_into_sentences(text):
    for w in split_into_words(s.lower()):
      yield (w, entry.filename)


Instruction:
Implement a Python 3 function for
phrasing demo map function.

Response:
def phrases_map(data):
  """Phrases demo map function."""
  (entry, text_fn) = data
  text = text_fn()
  filename = entry.filename

  logging.debug("Got %s", filename)
  for s in split_into_sentences(text):
    words = split_into_words(s.lower())
    if len(words) < PHRASE_LENGTH:
      yield (":".join(words), filename)
      continue
    for i in range(0, len(words) - PHRASE_LENGTH):
      yield (":".join(words[i:i+PHRASE_LENGTH]), filename)


Instruction:
How would you explain what the following Python 3 function does
def phrases_reduce(key, values):
  if len(values) < 10:
    return
  counts = {}
  for filename in values:
    counts[filename] = counts.get(filename, 0) + 1

  words = re.sub(r":", " ", key)
  threshold = len(values) / 2
  for filename, count in counts.items():
    if count > threshold:
      yield "%s:%s\n" % (words, filename)

Response: Phrases demo reduce function.


Instruction:
Can you create a Python 3 function that
returns the internal key for a particular item in the database.

Response:
def getKeyName(username, date, blob_key):
    """Returns the internal key for a particular item in the database.

    Our items are stored with keys of the form 'user/date/blob_key' ('/' is
    not the real separator, but __SEP is).

    Args:
      username: The given user's e-mail address.
      date: A datetime object representing the date and time that an input
        file was uploaded to this app.
      blob_key: The blob key corresponding to the location of the input file
        in the Blobstore.
    Returns:
      The internal key for the item specified by (username, date, blob_key).
    """

    sep = FileMetadata.__SEP
    return str(username + sep + str(date) + sep + blob_key)


Instruction:
Can you create a Python 3 function that
loads a Custodian instance from a dict containing a spec.

Response:
def from_spec(cls, spec):
        """
        Load a Custodian instance where the jobs are specified from a
        structure and a spec dict. This allows simple
        custom job sequences to be constructed quickly via a YAML file.

        Args:
            spec (dict): A dict specifying job. A sample of the dict in
                YAML format for the usual MP workflow is given as follows

                ```
                jobs:
                - jb: custodian.vasp.jobs.VaspJob
                  params:
                    final: False
                    suffix: .relax1
                - jb: custodian.vasp.jobs.VaspJob
                  params:
                    final: True
                    suffix: .relax2
                    settings_override: {"file": "CONTCAR", "action": {"_file_copy": {"dest": "POSCAR"}}
                jobs_common_params:
                  vasp_cmd: /opt/vasp
                handlers:
                - hdlr: custodian.vasp.handlers.VaspErrorHandler
                - hdlr: custodian.vasp.handlers.AliasingErrorHandler
                - hdlr: custodian.vasp.handlers.MeshSymmetryHandler
                validators:
                - vldr: custodian.vasp.validators.VasprunXMLValidator
                custodian_params:
                  scratch_dir: /tmp
                ```

                The `jobs` key is a list of jobs. Each job is
                specified via "job": <explicit path>, and all parameters are
                specified via `params` which is a dict.

                `common_params` specify a common set of parameters that are
                passed to all jobs, e.g., vasp_cmd.

        Returns:
            Custodian instance.
        """

        dec = MontyDecoder()

        def load_class(dotpath):
            modname, classname = dotpath.rsplit(".", 1)
            mod = __import__(modname, globals(), locals(), [classname], 0)
            return getattr(mod, classname)

        def process_params(d):
            decoded = {}
            for k, v in d.items():
                if k.startswith("$"):
                    if isinstance(v, list):
                        v = [os.path.expandvars(i) for i in v]
                    elif isinstance(v, dict):
                        v = {k2: os.path.expandvars(v2) for k2, v2 in v.items()}
                    else:
                        v = os.path.expandvars(v)
                decoded[k.strip("$")] = dec.process_decoded(v)
            return decoded

        jobs = []
        common_params = process_params(spec.get("jobs_common_params", {}))

        for d in spec["jobs"]:
            cls_ = load_class(d["jb"])
            params = process_params(d.get("params", {}))
            params.update(common_params)
            jobs.append(cls_(**params))

        handlers = []
        for d in spec.get("handlers", []):
            cls_ = load_class(d["hdlr"])
            params = process_params(d.get("params", {}))
            handlers.append(cls_(**params))

        validators = []
        for d in spec.get("validators", []):
            cls_ = load_class(d["vldr"])
            params = process_params(d.get("params", {}))
            validators.append(cls_(**params))

        custodian_params = process_params(spec.get("custodian_params", {}))

        return cls(jobs=jobs, handlers=handlers, validators=validators,
                   **custodian_params)


Instruction:
Make a summary of the following Python 3 code
def run(self):
        cwd = os.getcwd()

        with ScratchDir(self.scratch_dir, create_symbolic_link=True,
                        copy_to_current_on_exit=True,
                        copy_from_current_on_enter=True) as temp_dir:
            self.total_errors = 0
            start = datetime.datetime.now()
            logger.info("Run started at {} in {}.".format(
                start, temp_dir))
            v = sys.version.replace("\n", " ")
            logger.info("Custodian running on Python version {}".format(v))
            logger.info("Hostname: {}, Cluster: {}".format(
                *get_execution_host_info()))

            try:
                # skip jobs until the restart
                for job_n, job in islice(enumerate(self.jobs, 1),
                                         self.restart, None):
                    self._run_job(job_n, job)
                    # We do a dump of the run log after each job.
                    dumpfn(self.run_log, Custodian.LOG_FILE, cls=MontyEncoder,
                           indent=4)
                    # Checkpoint after each job so that we can recover from last
                    # point and remove old checkpoints
                    if self.checkpoint:
                        self.restart = job_n
                        Custodian._save_checkpoint(cwd, job_n)
            except CustodianError as ex:
                logger.error(ex.message)
                if ex.raises:
                    raise
            finally:
                # Log the corrections to a json file.
                logger.info("Logging to {}...".format(Custodian.LOG_FILE))
                dumpfn(self.run_log, Custodian.LOG_FILE, cls=MontyEncoder,
                       indent=4)
                end = datetime.datetime.now()
                logger.info("Run ended at {}.".format(end))
                run_time = end - start
                logger.info("Run completed. Total time taken = {}."
                            .format(run_time))
                if self.gzipped_output:
                    gzip_dir(".")

            # Cleanup checkpoint files (if any) if run is successful.
            Custodian._delete_checkpoints(cwd)

        return self.run_log

Response: Runs all jobs in the cluster and returns a list of lists of error dictionaries.


Instruction:
Can you tell what is the following Python 3 function doing
def _run_job(self, job_n, job):
        self.run_log.append({"job": job.as_dict(), "corrections": [],
                             "handler": None, "validator": None,
                             "max_errors": False, "max_errors_per_job": False,
                             "max_errors_per_handler": False,
                             "nonzero_return_code": False})
        self.errors_current_job = 0
        # reset the counters of the number of times a correction has been
        # applied for each handler
        for h in self.handlers:
            h.n_applied_corrections = 0

        job.setup()

        attempt = 0
        while (self.total_errors < self.max_errors and
               self.errors_current_job < self.max_errors_per_job):
            attempt += 1
            logger.info(
                "Starting job no. {} ({}) attempt no. {}. Total errors and "
                "errors in job thus far = {}, {}.".format(
                    job_n, job.name, attempt, self.total_errors,
                    self.errors_current_job))

            p = job.run()
            # Check for errors using the error handlers and perform
            # corrections.
            has_error = False
            zero_return_code = True

            # While the job is running, we use the handlers that are
            # monitors to monitor the job.
            if isinstance(p, subprocess.Popen):
                if self.monitors:
                    n = 0
                    while True:
                        n += 1
                        time.sleep(self.polling_time_step)
                        if p.poll() is not None:
                            break
                        terminate = self.terminate_func or p.terminate
                        if n % self.monitor_freq == 0:
                            has_error = self._do_check(self.monitors,
                                                       terminate)
                        if terminate is not None and terminate != p.terminate:
                            time.sleep(self.polling_time_step)
                else:
                    p.wait()
                    if self.terminate_func is not None and \
                            self.terminate_func != p.terminate:
                        self.terminate_func()
                        time.sleep(self.polling_time_step)

                zero_return_code = p.returncode == 0

            logger.info("{}.run has completed. "
                        "Checking remaining handlers".format(job.name))
            # Check for errors again, since in some cases non-monitor
            # handlers fix the problems detected by monitors
            # if an error has been found, not all handlers need to run
            if has_error:
                self._do_check([h for h in self.handlers
                                if not h.is_monitor])
            else:
                has_error = self._do_check(self.handlers)

            if has_error:
                # This makes sure the job is killed cleanly for certain systems.
                job.terminate()

            # If there are no errors detected, perform
            # postprocessing and exit.
            if not has_error:
                for v in self.validators:
                    if v.check():
                        self.run_log[-1]["validator"] = v
                        s = "Validation failed: {}".format(v)
                        raise ValidationError(s, True, v)
                if not zero_return_code:
                    if self.terminate_on_nonzero_returncode:
                        self.run_log[-1]["nonzero_return_code"] = True
                        s = "Job return code is %d. Terminating..." % \
                            p.returncode
                        logger.info(s)
                        raise ReturnCodeError(s, True)
                    else:
                        warnings.warn("subprocess returned a non-zero return "
                                      "code. Check outputs carefully...")
                job.postprocess()
                return

            # Check that all errors could be handled
            for x in self.run_log[-1]["corrections"]:
                if not x["actions"] and x["handler"].raises_runtime_error:
                    self.run_log[-1]["handler"] = x["handler"]
                    s = "Unrecoverable error for handler: {}".format(x["handler"])
                    raise NonRecoverableError(s, True, x["handler"])
            for x in self.run_log[-1]["corrections"]:
                if not x["actions"]:
                    self.run_log[-1]["handler"] = x["handler"]
                    s = "Unrecoverable error for handler: %s" % x["handler"]
                    raise NonRecoverableError(s, False, x["handler"])

        if self.errors_current_job >= self.max_errors_per_job:
            self.run_log[-1]["max_errors_per_job"] = True
            msg = "Max errors per job reached: {}.".format(self.max_errors_per_job)
            logger.info(msg)
            raise MaxCorrectionsPerJobError(msg, True, self.max_errors_per_job, job)
        else:
            self.run_log[-1]["max_errors"] = True
            msg = "Max errors reached: {}.".format(self.max_errors)
            logger.info(msg)
            raise MaxCorrectionsError(msg, True, self.max_errors)

Response: Runs a single job.


Instruction:
Implement a function in Python 3 to
run custodian in interuppted mode.

Response:
def run_interrupted(self):
        """
        Runs custodian in a interuppted mode, which sets up and
        validates jobs but doesn't run the executable

        Returns:
            number of remaining jobs

        Raises:
            ValidationError: if a job fails validation
            ReturnCodeError: if the process has a return code different from 0
            NonRecoverableError: if an unrecoverable occurs
            MaxCorrectionsPerJobError: if max_errors_per_job is reached
            MaxCorrectionsError: if max_errors is reached
            MaxCorrectionsPerHandlerError: if max_errors_per_handler is reached
        """
        start = datetime.datetime.now()
        try:
            cwd = os.getcwd()
            v = sys.version.replace("\n", " ")
            logger.info("Custodian started in singleshot mode at {} in {}."
                        .format(start, cwd))
            logger.info("Custodian running on Python version {}".format(v))

            # load run log
            if os.path.exists(Custodian.LOG_FILE):
                self.run_log = loadfn(Custodian.LOG_FILE, cls=MontyDecoder)

            if len(self.run_log) == 0:
                # starting up an initial job - setup input and quit
                job_n = 0
                job = self.jobs[job_n]
                logger.info("Setting up job no. 1 ({}) ".format(job.name))
                job.setup()
                self.run_log.append({"job": job.as_dict(), "corrections": [],
                                     'job_n': job_n})
                return len(self.jobs)
            else:
                # Continuing after running calculation
                job_n = self.run_log[-1]['job_n']
                job = self.jobs[job_n]

                # If we had to fix errors from a previous run, insert clean log
                # dict
                if len(self.run_log[-1]['corrections']) > 0:
                    logger.info("Reran {}.run due to fixable errors".format(
                        job.name))

                # check error handlers
                logger.info("Checking error handlers for {}.run".format(
                    job.name))
                if self._do_check(self.handlers):
                    logger.info("Failed validation based on error handlers")
                    # raise an error for an unrecoverable error
                    for x in self.run_log[-1]["corrections"]:
                        if not x["actions"] and x["handler"].raises_runtime_error:
                            self.run_log[-1]["handler"] = x["handler"]
                            s = "Unrecoverable error for handler: {}. " \
                                "Raising RuntimeError".format(x["handler"])
                            raise NonRecoverableError(s, True, x["handler"])
                    logger.info("Corrected input based on error handlers")
                    # Return with more jobs to run if recoverable error caught
                    # and corrected for
                    return len(self.jobs) - job_n

                # check validators
                logger.info("Checking validator for {}.run".format(job.name))
                for v in self.validators:
                    if v.check():
                        self.run_log[-1]["validator"] = v
                        logger.info("Failed validation based on validator")
                        s = "Validation failed: {}".format(v)
                        raise ValidationError(s, True, v)

                logger.info("Postprocessing for {}.run".format(job.name))
                job.postprocess()

                # IF DONE WITH ALL JOBS - DELETE ALL CHECKPOINTS AND RETURN
                # VALIDATED
                if len(self.jobs) == (job_n + 1):
                    self.finished = True
                    return 0

                # Setup next job_n
                job_n += 1
                job = self.jobs[job_n]
                self.run_log.append({"job": job.as_dict(), "corrections": [],
                                     'job_n': job_n})
                job.setup()
                return len(self.jobs) - job_n

        except CustodianError as ex:
            logger.error(ex.message)
            if ex.raises:
                raise

        finally:
            # Log the corrections to a json file.
            logger.info("Logging to {}...".format(Custodian.LOG_FILE))
            dumpfn(self.run_log, Custodian.LOG_FILE, cls=MontyEncoder,
                   indent=4)
            end = datetime.datetime.now()
            logger.info("Run ended at {}.".format(end))
            run_time = end - start
            logger.info("Run completed. Total time taken = {}."
                        .format(run_time))
            if self.finished and self.gzipped_output:
                gzip_dir(".")


Instruction:
Make a summary of the following Python 3 code
def _do_check(self, handlers, terminate_func=None):
        corrections = []
        for h in handlers:
            try:
                if h.check():
                    if h.max_num_corrections is not None \
                            and h.n_applied_corrections >= h.max_num_corrections:
                        msg = "Maximum number of corrections {} reached " \
                              "for handler {}".format(h.max_num_corrections, h)
                        if h.raise_on_max:
                            self.run_log[-1]["handler"] = h
                            self.run_log[-1]["max_errors_per_handler"] = True
                            raise MaxCorrectionsPerHandlerError(msg, True, h.max_num_corrections, h)
                        else:
                            logger.warning(msg+" Correction not applied.")
                            continue
                    if terminate_func is not None and h.is_terminating:
                        logger.info("Terminating job")
                        terminate_func()
                        # make sure we don't terminate twice
                        terminate_func = None
                    d = h.correct()
                    d["handler"] = h
                    logger.error("\n" + pformat(d, indent=2, width=-1))
                    corrections.append(d)
                    h.n_applied_corrections += 1
            except Exception:
                if not self.skip_over_errors:
                    raise
                else:
                    import traceback
                    logger.error("Bad handler %s " % h)
                    logger.error(traceback.format_exc())
                    corrections.append(
                        {"errors": ["Bad handler %s " % h],
                         "actions": []})
        self.total_errors += len(corrections)
        self.errors_current_job += len(corrections)
        self.run_log[-1]["corrections"].extend(corrections)
        # We do a dump of the run log after each check.
        dumpfn(self.run_log, Custodian.LOG_FILE, cls=MontyEncoder,
               indent=4)
        return len(corrections) > 0

Response: Checks the specified handlers and returns True iff errors caught.


Instruction:
Can you generate the documentation for the following Python 3 function
def apply_actions(self, actions):
        modified = []
        for a in actions:
            if "dict" in a:
                k = a["dict"]
                modified.append(k)
                self.feffinp[k] = self.modify_object(a["action"], self.feffinp[k])
            elif "file" in a:
                self.modify(a["action"], a["file"])
            else:
                raise ValueError("Unrecognized format: {}".format(a))
        if modified:
            feff = self.feffinp
            feff_input = "\n\n".join(str(feff[k]) for k in
                                     ["HEADER", "PARAMETERS", "POTENTIALS", "ATOMS"]
                                     if k in feff)
            for k, v in six.iteritems(feff):
                with open(os.path.join('.', k), "w") as f:
                    f.write(str(v))

            with open(os.path.join('.', "feff.inp"), "w") as f:
                f.write(feff_input)

Response:
Applies a list of actions to the FEFF Input Set and rewrites modified
            files.


Instruction:
Implement a function in Python 3 to
create a file. Args: filename (str): Filename. settings (dict): Must be {"content": actual_content}

Response:
def file_create(filename, settings):
        """
        Creates a file.

        Args:
            filename (str): Filename.
            settings (dict): Must be {"content": actual_content}
        """
        if len(settings) != 1:
            raise ValueError("Settings must only contain one item with key "
                             "'content'.")
        for k, v in settings.items():
            if k == "content":
                with open(filename, 'w') as f:
                    f.write(v)


Instruction:
Can you write a function in Python 3 where it
moves a file. {'_file_move': {'dest': 'new_file_name'}} Args: filename (str): Filename. settings (dict): Must be {"dest": path of new file}

Response:
def file_move(filename, settings):
        """
        Moves a file. {'_file_move': {'dest': 'new_file_name'}}

        Args:
            filename (str): Filename.
            settings (dict): Must be {"dest": path of new file}
        """
        if len(settings) != 1:
            raise ValueError("Settings must only contain one item with key "
                             "'dest'.")
        for k, v in settings.items():
            if k == "dest":
                shutil.move(filename, v)


Instruction:
Implement a function in Python 3 to
delete a file. {'_file_delete': {'mode': "actual"}} Args: filename (str): Filename. settings (dict): Must be {"mode": actual/simulated}. Simulated mode only prints the action without performing it.

Response:
def file_delete(filename, settings):
        """
        Deletes a file. {'_file_delete': {'mode': "actual"}}

        Args:
            filename (str): Filename.
            settings (dict): Must be {"mode": actual/simulated}. Simulated
                mode only prints the action without performing it.
        """
        if len(settings) != 1:
            raise ValueError("Settings must only contain one item with key "
                             "'mode'.")
        for k, v in settings.items():
            if k == "mode" and v == "actual":
                try:
                    os.remove(filename)
                except OSError:
                    #Skip file not found error.
                    pass
            elif k == "mode" and v == "simulated":
                print("Simulated removal of {}".format(filename))


Instruction:
Can you generate the documentation for the following Python 3 function
def file_copy(filename, settings):
        for k, v in settings.items():
            if k.startswith("dest"):
                shutil.copyfile(filename, v)

Response:
Copies a file. {'_file_copy': {'dest': 'new_file_name'}}

        Args:
            filename (str): Filename.
            settings (dict): Must be {"dest": path of new file}


Instruction:
Write a Python 3 script to
modify file with settings

Response:
def file_modify(filename, settings):
        """
        Modifies file access

        Args:
            filename (str): Filename.
            settings (dict): Can be "mode" or "owners"
        """
        for k, v in settings.items():
            if k == "mode":
                os.chmod(filename,v)
            if k == "owners":
                os.chown(filename,v)


Instruction:
Here you have a function in Python 3, explain what it does
def setup(self):
        decompress_dir('.')

        if self.backup:
            for f in FEFF_INPUT_FILES:
                shutil.copy(f, "{}.orig".format(f))

            for f in FEFF_BACKUP_FILES:
                if os.path.isfile(f):
                    shutil.copy(f, "{}.orig".format(f))

Response: Performs initial setup for FeffJob do backing up.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def run(self):

        """
        Performs the actual FEFF run
        Returns:
            (subprocess.Popen) Used for monitoring.
        """
        with open(self.output_file, "w") as f_std, \
                open(self.stderr_file, "w", buffering=1) as f_err:
            # Use line buffering for stderr
            # On TSCC, need to run shell command
            p = subprocess.Popen(self.feff_cmd, stdout=f_std, stderr=f_err, shell=True)

        return p

Response:
Performs the actual FEFF run
        Returns:
            (subprocess.Popen) Used for monitoring.


Instruction:
Given the following Python 3 function, write the documentation
def modify(self, modification, obj):
        for action, settings in modification.items():
            if action in self.supported_actions:
                self.supported_actions[action].__call__(obj, settings)
            elif self.strict:
                raise ValueError("{} is not a supported action!"
                                 .format(action))

Response: Modify the object in - place.


Instruction:
Implement a Python 3 function for
modifying an object that supports pymatgen s as_dict and from_dict API.

Response:
def modify_object(self, modification, obj):
        """
        Modify an object that supports pymatgen's as_dict() and from_dict API.

        Args:
            modification (dict): Modification must be {action_keyword :
                settings}. E.g., {'_set': {'Hello':'Universe', 'Bye': 'World'}}
            obj (object): Object to modify
        """
        d = obj.as_dict()
        self.modify(modification, d)
        return obj.from_dict(d)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def apply_actions(self, actions):
        modified = []
        for a in actions:
            if "dict" in a:
                k = a["dict"]
                modified.append(k)
                self.vi[k] = self.modify_object(a["action"], self.vi[k])
            elif "file" in a:
                self.modify(a["action"], a["file"])
            else:
                raise ValueError("Unrecognized format: {}".format(a))
        for f in modified:
            self.vi[f].write_file(f)

Response:
Applies a list of actions to the Vasp Input Set and rewrites modified
            files.


Instruction:
Given the following Python 3 function, write the documentation
def backup(filenames, prefix="error"):
    num = max([0] + [int(f.split(".")[1])
                     for f in glob("{}.*.tar.gz".format(prefix))])
    filename = "{}.{}.tar.gz".format(prefix, num + 1)
    logging.info("Backing up run to {}.".format(filename))
    with tarfile.open(filename, "w:gz") as tar:
        for fname in filenames:
            for f in glob(fname):
                tar.add(f)

Response: Backup a list of files to a tar. gz file.


Instruction:
Here you have a function in Python 3, explain what it does
def get_execution_host_info():
    host = os.environ.get('HOSTNAME', None)
    cluster = os.environ.get('SGE_O_HOST', None)
    if host is None:
        try:
            import socket
            host = host or socket.gethostname()
        except:
            pass
    return host or 'unknown', cluster or 'unknown'

Response: Tries to return a tuple describing the execution host.


Instruction:
Create a Python 3 function for
performing the actual QChem run.

Response:
def run(self):
        """
        Perform the actual QChem run.

        Returns:
            (subprocess.Popen) Used for monitoring.
        """
        qclog = open(self.qclog_file, 'w')
        p = subprocess.Popen(self.current_command, stdout=qclog)
        return p


Instruction:
Here you have a function in Python 3, explain what it does
def opt_with_frequency_flattener(cls,
                                     qchem_command,
                                     multimode="openmp",
                                     input_file="mol.qin",
                                     output_file="mol.qout",
                                     qclog_file="mol.qclog",
                                     max_iterations=10,
                                     max_molecule_perturb_scale=0.3,
                                     check_connectivity=True,
                                     **QCJob_kwargs):
        """
        Optimize a structure and calculate vibrational frequencies to check if the
        structure is in a true minima. If a frequency is negative, iteratively
        perturbe the geometry, optimize, and recalculate frequencies until all are
        positive, aka a true minima has been found.

        Args:
            qchem_command (str): Command to run QChem.
            multimode (str): Parallelization scheme, either openmp or mpi.
            input_file (str): Name of the QChem input file.
            output_file (str): Name of the QChem output file.
            max_iterations (int): Number of perturbation -> optimization -> frequency
                iterations to perform. Defaults to 10.
            max_molecule_perturb_scale (float): The maximum scaled perturbation that
                can be applied to the molecule. Defaults to 0.3.
            check_connectivity (bool): Whether to check differences in connectivity
                introduced by structural perturbation. Defaults to True.
            **QCJob_kwargs: Passthrough kwargs to QCJob. See
                :class:`custodian.qchem.jobs.QCJob`.
        """

        min_molecule_perturb_scale = 0.1
        scale_grid = 10
        perturb_scale_grid = (
            max_molecule_perturb_scale - min_molecule_perturb_scale
        ) / scale_grid

        if not os.path.exists(input_file):
            raise AssertionError('Input file must be present!')
        orig_opt_input = QCInput.from_file(input_file)
        orig_opt_rem = copy.deepcopy(orig_opt_input.rem)
        orig_freq_rem = copy.deepcopy(orig_opt_input.rem)
        orig_freq_rem["job_type"] = "freq"
        first = True
        reversed_direction = False
        num_neg_freqs = []

        for ii in range(max_iterations):
            yield (QCJob(
                qchem_command=qchem_command,
                multimode=multimode,
                input_file=input_file,
                output_file=output_file,
                qclog_file=qclog_file,
                suffix=".opt_" + str(ii),
                backup=first,
                **QCJob_kwargs))
            first = False
            opt_outdata = QCOutput(output_file + ".opt_" + str(ii)).data
            if opt_outdata["structure_change"] == "unconnected_fragments" and not opt_outdata["completion"]:
                print("Unstable molecule broke into unconnected fragments which failed to optimize! Exiting...")
                break
            else:
                freq_QCInput = QCInput(
                    molecule=opt_outdata.get("molecule_from_optimized_geometry"),
                    rem=orig_freq_rem,
                    opt=orig_opt_input.opt,
                    pcm=orig_opt_input.pcm,
                    solvent=orig_opt_input.solvent)
                freq_QCInput.write_file(input_file)
                yield (QCJob(
                    qchem_command=qchem_command,
                    multimode=multimode,
                    input_file=input_file,
                    output_file=output_file,
                    qclog_file=qclog_file,
                    suffix=".freq_" + str(ii),
                    backup=first,
                    **QCJob_kwargs))
                outdata = QCOutput(output_file + ".freq_" + str(ii)).data
                errors = outdata.get("errors")
                if len(errors) != 0:
                    raise AssertionError('No errors should be encountered while flattening frequencies!')
                if outdata.get('frequencies')[0] > 0.0:
                    print("All frequencies positive!")
                    break
                else:
                    num_neg_freqs += [sum(1 for freq in outdata.get('frequencies') if freq < 0)]
                    if len(num_neg_freqs) > 1:
                        if num_neg_freqs[-1] == num_neg_freqs[-2] and not reversed_direction:
                            reversed_direction = True
                        elif num_neg_freqs[-1] == num_neg_freqs[-2] and reversed_direction:
                            if len(num_neg_freqs) < 3:
                                raise AssertionError("ERROR: This should only be possible after at least three frequency flattening iterations! Exiting...")
                            else:
                                raise Exception("ERROR: Reversing the perturbation direction still could not flatten any frequencies. Exiting...")
                        elif num_neg_freqs[-1] != num_neg_freqs[-2] and reversed_direction:
                            reversed_direction = False

                    negative_freq_vecs = outdata.get("frequency_mode_vectors")[0]
                    structure_successfully_perturbed = False

                    for molecule_perturb_scale in np.arange(
                            max_molecule_perturb_scale, min_molecule_perturb_scale,
                            -perturb_scale_grid):
                        new_coords = perturb_coordinates(
                            old_coords=outdata.get("initial_geometry"),
                            negative_freq_vecs=negative_freq_vecs,
                            molecule_perturb_scale=molecule_perturb_scale,
                            reversed_direction=reversed_direction)
                        new_molecule = Molecule(
                            species=outdata.get('species'),
                            coords=new_coords,
                            charge=outdata.get('charge'),
                            spin_multiplicity=outdata.get('multiplicity'))
                        if check_connectivity:
                            old_molgraph = MoleculeGraph.with_local_env_strategy(outdata.get("initial_molecule"),
                                                               OpenBabelNN(),
                                                               reorder=False,
                                                               extend_structure=False)
                            new_molgraph = MoleculeGraph.with_local_env_strategy(new_molecule,
                                                               OpenBabelNN(),
                                                               reorder=False,
                                                               extend_structure=False)
                            if old_molgraph.isomorphic_to(new_molgraph):
                                structure_successfully_perturbed = True
                                break
                    if not structure_successfully_perturbed:
                        raise Exception(
                            "ERROR: Unable to perturb coordinates to remove negative frequency without changing the connectivity! Exiting..."
                        )

                    new_opt_QCInput = QCInput(
                        molecule=new_molecule,
                        rem=orig_opt_rem,
                        opt=orig_opt_input.opt,
                        pcm=orig_opt_input.pcm,
                        solvent=orig_opt_input.solvent)
                    new_opt_QCInput.write_file(input_file)

Response: Optimize a structure and calculate vibrational frequencies.


Instruction:
Given the following Python 3 function, write the documentation
def setup(self):
        decompress_dir('.')

        if self.backup:
            for f in VASP_INPUT_FILES:
                shutil.copy(f, "{}.orig".format(f))

        if self.auto_npar:
            try:
                incar = Incar.from_file("INCAR")
                # Only optimized NPAR for non-HF and non-RPA calculations.
                if not (incar.get("LHFCALC") or incar.get("LRPA") or
                        incar.get("LEPSILON")):
                    if incar.get("IBRION") in [5, 6, 7, 8]:
                        # NPAR should not be set for Hessian matrix
                        # calculations, whether in DFPT or otherwise.
                        del incar["NPAR"]
                    else:
                        import multiprocessing
                        # try sge environment variable first
                        # (since multiprocessing counts cores on the current
                        # machine only)
                        ncores = os.environ.get('NSLOTS') or \
                            multiprocessing.cpu_count()
                        ncores = int(ncores)
                        for npar in range(int(math.sqrt(ncores)),
                                          ncores):
                            if ncores % npar == 0:
                                incar["NPAR"] = npar
                                break
                    incar.write_file("INCAR")
            except:
                pass

        if self.auto_continue:
            if os.path.exists("continue.json"):
                actions = loadfn("continue.json").get("actions")
                logger.info("Continuing previous VaspJob. Actions: {}".format(actions))
                backup(VASP_BACKUP_FILES, prefix="prev_run")
                VaspModder().apply_actions(actions)

            else:
                # Default functionality is to copy CONTCAR to POSCAR and set
                # ISTART to 1 in the INCAR, but other actions can be specified
                if self.auto_continue is True:
                    actions = [{"file": "CONTCAR",
                                "action": {"_file_copy": {"dest": "POSCAR"}}},
                               {"dict": "INCAR",
                                "action": {"_set": {"ISTART": 1}}}]
                else:
                    actions = self.auto_continue
                dumpfn({"actions": actions}, "continue.json")

        if self.settings_override is not None:
            VaspModder().apply_actions(self.settings_override)

Response: Sets up the current state of the VASP job.


Instruction:
Can you create a Python 3 function that
performs the actual VASP run.

Response:
def run(self):
        """
        Perform the actual VASP run.

        Returns:
            (subprocess.Popen) Used for monitoring.
        """
        cmd = list(self.vasp_cmd)
        if self.auto_gamma:
            vi = VaspInput.from_directory(".")
            kpts = vi["KPOINTS"]
            if kpts.style == Kpoints.supported_modes.Gamma \
                    and tuple(kpts.kpts[0]) == (1, 1, 1):
                if self.gamma_vasp_cmd is not None and which(
                        self.gamma_vasp_cmd[-1]):
                    cmd = self.gamma_vasp_cmd
                elif which(cmd[-1] + ".gamma"):
                    cmd[-1] += ".gamma"
        logger.info("Running {}".format(" ".join(cmd)))
        with open(self.output_file, 'w') as f_std, \
                open(self.stderr_file, "w", buffering=1) as f_err:
            # use line buffering for stderr
            p = subprocess.Popen(cmd, stdout=f_std, stderr=f_err)
        return p


Instruction:
Can you generate the documentation for the following Python 3 function
def postprocess(self):
        for f in VASP_OUTPUT_FILES + [self.output_file]:
            if os.path.exists(f):
                if self.final and self.suffix != "":
                    shutil.move(f, "{}{}".format(f, self.suffix))
                elif self.suffix != "":
                    shutil.copy(f, "{}{}".format(f, self.suffix))

        if self.copy_magmom and not self.final:
            try:
                outcar = Outcar("OUTCAR")
                magmom = [m['tot'] for m in outcar.magnetization]
                incar = Incar.from_file("INCAR")
                incar['MAGMOM'] = magmom
                incar.write_file("INCAR")
            except:
                logger.error('MAGMOM copy from OUTCAR to INCAR failed')

        # Remove continuation so if a subsequent job is run in
        # the same directory, will not restart this job.
        if os.path.exists("continue.json"):
            os.remove("continue.json")

Response: Postprocessing includes renaming gzipping where necessary.


Instruction:
Can you generate the documentation for the following Python 3 function
def double_relaxation_run(cls, vasp_cmd, auto_npar=True, ediffg=-0.05,
                              half_kpts_first_relax=False, auto_continue=False):
        """
        Returns a list of two jobs corresponding to an AFLOW style double
        relaxation run.

        Args:
            vasp_cmd (str): Command to run vasp as a list of args. For example,
                if you are using mpirun, it can be something like
                ["mpirun", "pvasp.5.2.11"]
            auto_npar (bool): Whether to automatically tune NPAR to be sqrt(
                number of cores) as recommended by VASP for DFT calculations.
                Generally, this results in significant speedups. Defaults to
                True. Set to False for HF, GW and RPA calculations.
            ediffg (float): Force convergence criteria for subsequent runs (
                ignored for the initial run.)
            half_kpts_first_relax (bool): Whether to halve the kpoint grid
                for the first relaxation. Speeds up difficult convergence
                considerably. Defaults to False.

        Returns:
            List of two jobs corresponding to an AFLOW style run.
        """
        incar_update = {"ISTART": 1}
        if ediffg:
            incar_update["EDIFFG"] = ediffg
        settings_overide_1 = None
        settings_overide_2 = [
            {"dict": "INCAR",
             "action": {"_set": incar_update}},
            {"file": "CONTCAR",
             "action": {"_file_copy": {"dest": "POSCAR"}}}]
        if half_kpts_first_relax and os.path.exists("KPOINTS") and \
                os.path.exists("POSCAR"):
            kpts = Kpoints.from_file("KPOINTS")
            orig_kpts_dict = kpts.as_dict()
            # lattice vectors with length < 8 will get >1 KPOINT
            kpts.kpts = np.round(np.maximum(np.array(kpts.kpts) / 2,
                                            1)).astype(int).tolist()
            low_kpts_dict = kpts.as_dict()
            settings_overide_1 = [
                {"dict": "KPOINTS",
                 "action": {"_set": low_kpts_dict}}
            ]
            settings_overide_2.append(
                {"dict": "KPOINTS",
                 "action": {"_set": orig_kpts_dict}}
            )

        return [VaspJob(vasp_cmd, final=False, suffix=".relax1",
                        auto_npar=auto_npar, auto_continue=auto_continue,
                        settings_override=settings_overide_1),
                VaspJob(vasp_cmd, final=True, backup=False, suffix=".relax2",
                        auto_npar=auto_npar, auto_continue=auto_continue,
                        settings_override=settings_overide_2)]

Response: Returns a list of two jobs corresponding to a double relaxation run.


Instruction:
Can you tell what is the following Python 3 function doing
def metagga_opt_run(cls, vasp_cmd, auto_npar=True, ediffg=-0.05,
                        half_kpts_first_relax=False, auto_continue=False):
        """
        Returns a list of thres jobs to perform an optimization for any
        metaGGA functional. There is an initial calculation of the
        GGA wavefunction which is fed into the initial metaGGA optimization
        to precondition the electronic structure optimizer. The metaGGA
        optimization is performed using the double relaxation scheme
        """

        incar = Incar.from_file("INCAR")
        # Defaults to using the SCAN metaGGA
        metaGGA = incar.get("METAGGA", "SCAN")

        # Pre optimze WAVECAR and structure using regular GGA
        pre_opt_setings = [{"dict": "INCAR",
                            "action": {"_set": {"METAGGA": None,
                                                "LWAVE": True,
                                                "NSW": 0}}}]
        jobs = [VaspJob(vasp_cmd, auto_npar=auto_npar,
                        final=False, suffix=".precondition",
                        settings_override=pre_opt_setings)]

        # Finish with regular double relaxation style run using SCAN
        jobs.extend(VaspJob.double_relaxation_run(vasp_cmd, auto_npar=auto_npar,
                                                  ediffg=ediffg,
                                                  half_kpts_first_relax=half_kpts_first_relax))

        # Ensure the first relaxation doesn't overwrite the original inputs
        jobs[1].backup = False

        # Update double_relaxation job to start from pre-optimized run
        post_opt_settings = [{"dict": "INCAR",
                              "action": {"_set": {"METAGGA": metaGGA, "ISTART": 1,
                                                  "NSW": incar.get("NSW", 99),
                                                  "LWAVE": incar.get("LWAVE", False)}}},
                             {"file": "CONTCAR",
                              "action": {"_file_copy": {"dest": "POSCAR"}}}]
        if jobs[1].settings_override:
            post_opt_settings = jobs[1].settings_override + post_opt_settings
        jobs[1].settings_override = post_opt_settings

        return jobs

Response: Performs an optimization for any specific metaGGA functional.