Instruction:
Implement a function in Python 3 to
process a given path and returns a list of paths with environment variables replaced with their respective values.

Response:
def Process(self, path):
    """Processes a given path.

    Args:
      path: Path (as a string) to post-process.

    Returns:
      A list of paths with environment variables replaced with their
      values. If the mapping had a list of values for a particular variable,
      instead of just one value, then all possible replacements will be
      returned.
    """
    path = re.sub(self.SYSTEMROOT_RE, r"%systemroot%", path, count=1)
    path = re.sub(self.SYSTEM32_RE, r"%systemroot%\\system32", path, count=1)

    matches_iter = self.WIN_ENVIRON_REGEX.finditer(path)
    var_names = set(m.group(1).lower() for m in matches_iter)

    results = [path]
    for var_name in var_names:
      try:
        var_regex, var_value = self.vars_map[var_name]
      except KeyError:
        continue

      if isinstance(var_value, string_types):
        replacements = [var_value]
      else:
        replacements = var_value

      processed_results = []
      for result in results:
        for repl in replacements:
          # Using lambda here, as otherwise Python interprets \\f as a
          # backreference (same applies to \\0 and \\1). When using a
          # function as a replacement argument, backreferences are ignored.
          # pylint: disable=cell-var-from-loop
          processed_results.append(var_regex.sub(lambda _: repl, result))
      results = processed_results

    return results


Instruction:
Given the following Python 3 function, write the documentation
def CheckDataStoreAccess(self, token, subjects, requested_access="r"):
    logging.debug("Checking %s: %s for %s", token, subjects, requested_access)
    raise NotImplementedError()

Response: This function is called by the ACLToken object when the user is authorized to access the AFF4 resources.


Instruction:
Implement a Python 3 function for
writing hunt results from a given client as part of a given hunt.

Response:
def WriteHuntResults(client_id, hunt_id, responses):
  """Writes hunt results from a given client as part of a given hunt."""

  if not hunt.IsLegacyHunt(hunt_id):
    data_store.REL_DB.WriteFlowResults(responses)

    hunt.StopHuntIfCPUOrNetworkLimitsExceeded(hunt_id)
    return

  hunt_id_urn = rdfvalue.RDFURN("hunts").Add(hunt_id)

  msgs = []
  for response in responses:
    if isinstance(response, rdf_flow_objects.FlowStatus):
      continue

    msgs.append(
        rdf_flows.GrrMessage(payload=response.payload, source=client_id))

  with data_store.DB.GetMutationPool() as pool:
    for msg in msgs:
      hunts_results.HuntResultCollection.StaticAdd(
          hunt_id_urn.Add("Results"), msg, mutation_pool=pool)

    for msg in msgs:
      multi_type_collection.MultiTypeCollection.StaticAdd(
          hunt_id_urn.Add("ResultsPerType"), msg, mutation_pool=pool)

  stats_collector_instance.Get().IncrementCounter(
      "hunt_results_added", delta=len(responses))


Instruction:
Can you generate the documentation for the following Python 3 function
def ProcessHuntFlowError(flow_obj,
                         error_message=None,
                         backtrace=None,
                         status_msg=None):
  """Processes error and status message for a given hunt-induced flow."""

  if not hunt.IsLegacyHunt(flow_obj.parent_hunt_id):
    hunt.StopHuntIfCPUOrNetworkLimitsExceeded(flow_obj.parent_hunt_id)
    return

  hunt_urn = rdfvalue.RDFURN("hunts").Add(flow_obj.parent_hunt_id)
  client_urn = rdf_client.ClientURN(flow_obj.client_id)

  error = rdf_hunts.HuntError(client_id=flow_obj.client_id, backtrace=backtrace)
  if error_message is not None:
    error.log_message = error_message
  with data_store.DB.GetMutationPool() as pool:
    grr_collections.HuntErrorCollection.StaticAdd(
        hunt_urn.Add("ErrorClients"), error, mutation_pool=pool)
    grr_collections.ClientUrnCollection.StaticAdd(
        hunt_urn.Add("CompletedClients"), client_urn, mutation_pool=pool)

  if status_msg is not None:
    with aff4.FACTORY.Open(hunt_urn, mode="rw") as fd:
      # Legacy AFF4 code expects token to be set.
      fd.token = access_control.ACLToken(username=fd.creator)
      fd.GetRunner().SaveResourceUsage(flow_obj.client_id, status_msg)

Response: Processes an error message and status message for a given hunt - induced flow.


Instruction:
How would you explain what the following Python 3 function does
def ProcessHuntFlowDone(flow_obj, status_msg=None):

  if not hunt.IsLegacyHunt(flow_obj.parent_hunt_id):
    hunt_obj = hunt.StopHuntIfCPUOrNetworkLimitsExceeded(
        flow_obj.parent_hunt_id)
    hunt.CompleteHuntIfExpirationTimeReached(hunt_obj)
    return

  hunt_urn = rdfvalue.RDFURN("hunts").Add(flow_obj.parent_hunt_id)
  client_urn = rdf_client.ClientURN(flow_obj.client_id)

  # Update the counter metrics separately from collections to minimize
  # contention.
  with aff4.FACTORY.Open(hunt_urn, mode="rw") as fd:
    # Legacy AFF4 code expects token to be set.
    fd.token = access_control.ACLToken(username=fd.creator)

    if flow_obj.num_replies_sent:
      fd.context.clients_with_results_count += 1

    fd.context.completed_clients_count += 1
    fd.context.results_count += flow_obj.num_replies_sent

    fd.GetRunner().SaveResourceUsage(flow_obj.client_id, status_msg)

  with aff4.FACTORY.Open(hunt_urn, mode="rw") as fd:
    # Legacy AFF4 code expects token to be set.
    fd.token = access_control.ACLToken(username=fd.creator)

    fd.RegisterCompletedClient(client_urn)
    if flow_obj.num_replies_sent:
      fd.RegisterClientWithResults(client_urn)

    fd.StopHuntIfAverageLimitsExceeded()

Response: Processes a given hunt - induced flow.


Instruction:
Can you tell what is the following Python 3 function doing
def ProcessHuntFlowLog(flow_obj, log_msg):

  if not hunt.IsLegacyHunt(flow_obj.parent_hunt_id):
    return

  hunt_urn = rdfvalue.RDFURN("hunts").Add(flow_obj.parent_hunt_id)
  flow_urn = hunt_urn.Add(flow_obj.flow_id)
  log_entry = rdf_flows.FlowLog(
      client_id=flow_obj.client_id,
      urn=flow_urn,
      flow_name=flow_obj.flow_class_name,
      log_message=log_msg)
  with data_store.DB.GetMutationPool() as pool:
    grr_collections.LogCollection.StaticAdd(
        hunt_urn.Add("Logs"), log_entry, mutation_pool=pool)

Response: Processes a given hunt - induced flow log message.


Instruction:
Given the following Python 3 function, write the documentation
def ProcessHuntClientCrash(flow_obj, client_crash_info):

  if not hunt.IsLegacyHunt(flow_obj.parent_hunt_id):
    hunt.StopHuntIfCrashLimitExceeded(flow_obj.parent_hunt_id)
    return

  hunt_urn = rdfvalue.RDFURN("hunts").Add(flow_obj.parent_hunt_id)

  with aff4.FACTORY.Open(hunt_urn, mode="rw") as fd:
    # Legacy AFF4 code expects token to be set.
    fd.token = access_control.ACLToken(username=fd.creator)
    fd.RegisterCrash(client_crash_info)

Response: Processes a client crash triggered by a given hunt - induced flow.


Instruction:
Here you have a function in Python 3, explain what it does
def ValidateVfsPath(path):

  components = (path or "").lstrip("/").split("/")
  if not components:
    raise ValueError("Empty path is not a valid path: %s." %
                     utils.SmartStr(path))

  if components[0] not in ROOT_FILES_WHITELIST:
    raise ValueError(
        "First path component was '%s', but has to be one of %s" %
        (utils.SmartStr(components[0]), ", ".join(ROOT_FILES_WHITELIST)))

  return True

Response: Validates a VFS path.


Instruction:
Can you generate the documentation for the following Python 3 function
def _GenerateApiFileDetails(path_infos):

  type_attrs = []
  hash_attrs = []
  size_attrs = []
  stat_attrs = []
  pathspec_attrs = []

  def _Value(age, value):
    """Generate ApiAff4ObjectAttributeValue from an age and a value."""

    v = ApiAff4ObjectAttributeValue()
    # TODO(user): get rid of RDF builtin "age" property.
    v.Set("age", age)
    # With dynamic values we first have to set the type and
    # then the value itself.
    # TODO(user): refactor dynamic values logic so that it's not needed,
    # possibly just removing the "type" attribute completely.
    v.Set("type", compatibility.GetName(value.__class__))
    v.value = value
    return v

  for pi in path_infos:
    if pi.directory:
      object_type = "VFSDirectory"
    else:
      object_type = "VFSFile"

    type_attrs.append(_Value(pi.timestamp, rdfvalue.RDFString(object_type)))

    if pi.hash_entry:
      hash_attrs.append(_Value(pi.timestamp, pi.hash_entry))
      size_attrs.append(
          _Value(pi.timestamp, rdfvalue.RDFInteger(pi.hash_entry.num_bytes)))
    if pi.stat_entry:
      stat_attrs.append(_Value(pi.timestamp, pi.stat_entry))

      if pi.stat_entry.pathspec:
        pathspec_attrs.append(_Value(pi.timestamp, pi.stat_entry.pathspec))

  return ApiAff4ObjectRepresentation(types=[
      ApiAff4ObjectType(
          name="AFF4Object",
          attributes=[
              ApiAff4ObjectAttribute(
                  name="TYPE",
                  values=type_attrs,
              ),
          ],
      ),
      ApiAff4ObjectType(
          name="AFF4Stream",
          attributes=[
              ApiAff4ObjectAttribute(
                  name="HASH",
                  values=hash_attrs,
              ),
              ApiAff4ObjectAttribute(
                  name="SIZE",
                  values=size_attrs,
              ),
          ],
      ),
      ApiAff4ObjectType(
          name="VFSFile",
          attributes=[
              ApiAff4ObjectAttribute(
                  name="PATHSPEC",
                  values=pathspec_attrs,
              ),
              ApiAff4ObjectAttribute(
                  name="STAT",
                  values=stat_attrs,
              ),
          ],
      ),
  ])

Response: Generates ApiAff4ObjectRepresentation for a list of path infos.


Instruction:
How would you implement a function in Python 3 that
retrieves the total size in bytes of an AFF4 object.

Response:
def _Aff4Size(aff4_obj):
  """Retrieves the total size in bytes of an AFF4 object.

  Args:
    aff4_obj: An AFF4 stream instance to retrieve size for.

  Returns:
    An integer representing number of bytes.

  Raises:
    TypeError: If `aff4_obj` is not an instance of AFF4 stream.
  """
  if not isinstance(aff4_obj, aff4.AFF4Stream):
    message = "Expected an instance of `%s` but received `%s`"
    raise TypeError(message % (aff4.AFF4Stream, type(aff4_obj)))

  return int(aff4_obj.Get(aff4_obj.Schema.SIZE))


Instruction:
Write a Python 3 script to
read contents of given AFF4 stream.

Response:
def _Aff4Read(aff4_obj, offset, length):
  """Reads contents of given AFF4 file.

  Args:
    aff4_obj: An AFF4 stream instance to retrieve contents for.
    offset: An offset to start the reading from.
    length: A number of bytes to read. Reads the whole file if 0.

  Returns:
    Contents of specified AFF4 stream.

  Raises:
    TypeError: If `aff4_obj` is not an instance of AFF4 stream.
  """
  length = length or (_Aff4Size(aff4_obj) - offset)

  aff4_obj.Seek(offset)
  return aff4_obj.Read(length)


Instruction:
Implement a function in Python 3 to
retrieve timeline entries from AFF4.

Response:
def _GetTimelineStatEntriesLegacy(client_id, file_path, with_history=True):
  """Gets timeline entries from AFF4."""

  folder_urn = aff4.ROOT_URN.Add(str(client_id)).Add(file_path)

  child_urns = []
  for _, children in aff4.FACTORY.RecursiveMultiListChildren([folder_urn]):
    child_urns.extend(children)

  if with_history:
    timestamp = aff4.ALL_TIMES
  else:
    timestamp = aff4.NEWEST_TIME

  for fd in aff4.FACTORY.MultiOpen(child_urns, age=timestamp):
    file_path = "/".join(str(fd.urn).split("/")[2:])

    if not with_history:
      yield file_path, fd.Get(fd.Schema.STAT), fd.Get(fd.Schema.HASH)
      continue

    result = {}

    stats = fd.GetValuesForAttribute(fd.Schema.STAT)
    for s in stats:
      result[s.age] = [s, None]

    hashes = fd.GetValuesForAttribute(fd.Schema.HASH)
    for h in hashes:
      prev = result.setdefault(h.age, [None, None])
      prev[1] = h

    for ts in sorted(result):
      v = result[ts]
      yield file_path, v[0], v[1]


Instruction:
Can you implement a function in Python 3 that
gets timeline entries from REL_DB.

Response:
def _GetTimelineStatEntriesRelDB(api_client_id, file_path, with_history=True):
  """Gets timeline entries from REL_DB."""
  path_type, components = rdf_objects.ParseCategorizedPath(file_path)

  client_id = str(api_client_id)

  try:
    root_path_info = data_store.REL_DB.ReadPathInfo(client_id, path_type,
                                                    components)
  except db.UnknownPathError:
    return

  path_infos = []
  for path_info in itertools.chain(
      [root_path_info],
      data_store.REL_DB.ListDescendentPathInfos(client_id, path_type,
                                                components),
  ):
    # TODO(user): this is to keep the compatibility with current
    # AFF4 implementation. Check if this check is needed.
    if path_info.directory:
      continue

    categorized_path = rdf_objects.ToCategorizedPath(path_info.path_type,
                                                     path_info.components)
    if with_history:
      path_infos.append(path_info)
    else:
      yield categorized_path, path_info.stat_entry, path_info.hash_entry

  if with_history:
    hist_path_infos = data_store.REL_DB.ReadPathInfosHistories(
        client_id, path_type, [tuple(pi.components) for pi in path_infos])
    for path_info in itertools.chain(*hist_path_infos.itervalues()):
      categorized_path = rdf_objects.ToCategorizedPath(path_info.path_type,
                                                       path_info.components)
      yield categorized_path, path_info.stat_entry, path_info.hash_entry


Instruction:
Implement a Python 3 function for
retrieving timeline entries from the appropriate data source.

Response:
def _GetTimelineStatEntries(client_id, file_path, with_history=True):
  """Gets timeline entries from the appropriate data source (AFF4 or REL_DB)."""

  if data_store.RelationalDBEnabled():
    fn = _GetTimelineStatEntriesRelDB
  else:
    fn = _GetTimelineStatEntriesLegacy

  for v in fn(client_id, file_path, with_history=with_history):
    yield v


Instruction:
Can you write a function in Python 3 where it
returns a list of ApiVfsTimelineItem objects for a given client id and path.

Response:
def _GetTimelineItems(client_id, file_path):
  """Gets timeline items for a given client id and path."""

  items = []

  for file_path, stat_entry, _ in _GetTimelineStatEntries(
      client_id, file_path, with_history=True):

    # It may be that for a given timestamp only hash entry is available, we're
    # skipping those.
    if stat_entry is None:
      continue

    # Add a new event for each MAC time if it exists.
    for c in "mac":
      timestamp = getattr(stat_entry, "st_%stime" % c)
      if timestamp is None:
        continue

      item = ApiVfsTimelineItem()
      item.timestamp = rdfvalue.RDFDatetime.FromSecondsSinceEpoch(timestamp)

      # Remove aff4:/<client_id> to have a more concise path to the
      # subject.
      item.file_path = file_path
      if c == "m":
        item.action = ApiVfsTimelineItem.FileActionType.MODIFICATION
      elif c == "a":
        item.action = ApiVfsTimelineItem.FileActionType.ACCESS
      elif c == "c":
        item.action = ApiVfsTimelineItem.FileActionType.METADATA_CHANGED

      items.append(item)

  return sorted(items, key=lambda x: x.timestamp, reverse=True)


Instruction:
Can you tell what is the following Python 3 function doing
def InitFromAff4Object(self, aff4_obj, aff4_cls, attr_blacklist):
    self.name = str(aff4_cls.__name__)
    self.attributes = []

    schema = aff4_cls.SchemaCls
    for name, attribute in sorted(iteritems(schema.__dict__)):
      if not isinstance(attribute, aff4.Attribute):
        continue

      if name in attr_blacklist:
        continue

      attr_repr = ApiAff4ObjectAttribute()
      attr_repr.name = name
      attr_repr.description = attribute.description
      attr_repr.values = []

      values = list(aff4_obj.GetValuesForAttribute(attribute))
      for value in values:
        # This value is really a LazyDecoder() instance. We need to get at the
        # real data here.
        # TODO(user): Change GetValuesForAttribute to resolve
        # lazy decoders and directly yield the rdf value.
        if hasattr(value, "ToRDFValue"):
          value = value.ToRDFValue()

        value_repr = ApiAff4ObjectAttributeValue()
        value_repr.Set("type", compatibility.GetName(value.__class__))
        value_repr.Set("age", value.age)
        value_repr.value = value
        attr_repr.values.append(value_repr)

      if attr_repr.values:
        self.attributes.append(attr_repr)

    return self

Response: Initializes the current instance from an Aff4Object.


Instruction:
Can you tell what is the following Python 3 function doing
def InitFromAff4Object(self, aff4_obj):
    attr_blacklist = []  # We use this to show attributes only once.

    self.types = []
    for aff4_cls in aff4_obj.__class__.__mro__:
      if not hasattr(aff4_cls, "SchemaCls"):
        continue

      type_repr = ApiAff4ObjectType().InitFromAff4Object(
          aff4_obj, aff4_cls, attr_blacklist)

      if type_repr.attributes:
        self.types.append(type_repr)

      # Add all attribute names from this type representation to the
      # blacklist to not add them to the result again.
      attr_blacklist.extend([attr.name for attr in type_repr.attributes])

    return self

Response: Initializes the current instance from an Aff4Object.


Instruction:
Create a Python 3 function to
initialize the current instance from an Aff4Object.

Response:
def InitFromAff4Object(self,
                         file_obj,
                         stat_entry=None,
                         hash_entry=None,
                         with_details=False):
    """Initializes the current instance from an Aff4Stream.

    Args:
      file_obj: An Aff4Stream representing a file.
      stat_entry: An optional stat entry object to be used. If none is provided,
        the one stored in the AFF4 data store is used.
      hash_entry: An optional hash entry object to be used. If none is provided,
        the one stored in the AFF4 data store is used.
      with_details: True if all details of the Aff4Object should be included,
        false otherwise.

    Returns:
      A reference to the current instance.
    """
    self.name = file_obj.urn.Basename()
    self.path = "/".join(file_obj.urn.Path().split("/")[2:])
    self.is_directory = "Container" in file_obj.behaviours

    self.stat = stat_entry or file_obj.Get(file_obj.Schema.STAT)
    self.hash = hash_entry or file_obj.Get(file_obj.Schema.HASH, None)

    if not self.is_directory:
      try:
        self.last_collected = file_obj.GetContentAge()
      except AttributeError:
        # Defensive approach - in case file-like object doesn't have
        # GetContentAge defined.
        logging.debug("File-like object %s doesn't have GetContentAge defined.",
                      file_obj.__class__.__name__)

      if self.last_collected:
        self.last_collected_size = file_obj.Get(file_obj.Schema.SIZE)

    type_obj = file_obj.Get(file_obj.Schema.TYPE)
    if type_obj is not None:
      self.age = type_obj.age

    if with_details:
      self.details = ApiAff4ObjectRepresentation().InitFromAff4Object(file_obj)

    return self


Instruction:
Here you have a function in Python 3, explain what it does
def _Decode(self, codec_name, data):
    try:
      return data.decode(codec_name, "replace")
    except LookupError:
      raise RuntimeError("Codec could not be found.")
    except AssertionError:
      raise RuntimeError("Codec failed to decode")

Response: Decode data with the given codec name.


Instruction:
Can you create a Python 3 function that
signs multiple files at once.

Response:
def SignFiles(self, filenames):
    """Signs multiple files at once."""
    file_list = " ".join(filenames)
    subprocess.check_call("%s %s" % (self._signing_cmdline, file_list))
    if self._verification_cmdline:
      subprocess.check_call("%s %s" % (self._verification_cmdline, file_list))


Instruction:
Write a Python 3 script to
sign a buffer via temp files.

Response:
def SignBuffer(self, in_buffer):
    """Sign a buffer via temp files.

    Our signing tool can't sign a buffer, so we work around it using temporary
    files.

    Args:
      in_buffer: data to sign

    Returns:
      signed data
    """
    precondition.AssertType(in_buffer, bytes)
    with tempfile.NamedTemporaryFile() as temp_in:
      temp_in.write(in_buffer)
      temp_in.seek(0)
      outfile = self.SignFile(temp_in.name)
      with io.open(outfile, "rb") as filedesc:
        return filedesc.read()


Instruction:
How would you explain what the following Python 3 function does
def SignFile(self, in_filename, out_filename=None):
    if out_filename is None:
      out_filename = "%s.signed" % in_filename

    args = [
        "-certs", self.cert, "-key", self.key, "-n", self.application, "-t",
        "http://timestamp.verisign.com/scripts/timestamp.dll", "-h", "sha1",
        "-in", in_filename, "-out", out_filename
    ]

    try:
      output_log = io.StringIO()
      ossl = pexpect.spawn("osslsigncode", args)
      # Use logfile_read so we don't get the password we're injecting.
      ossl.logfile_read = output_log
      ossl.expect("Enter PEM pass phrase")
      ossl.sendline(self.password)
      ossl.wait()
    except pexpect.ExceptionPexpect:
      output_log.seek(0)
      logging.exception(output_log.read())
      raise

    if not os.path.exists(out_filename):
      raise SigningError("Expected output %s not created" % out_filename)

    try:
      subprocess.check_call(["osslsigncode", "verify", "-in", out_filename])
    except subprocess.CalledProcessError:
      logging.exception("Bad signature verification on %s", out_filename)
      raise SigningError("Bad signature verification on %s" % out_filename)

    return out_filename

Response: Signs a file using osslsigncode.


Instruction:
Can you generate the documentation for the following Python 3 function
def AddSignatureToRPMs(self, rpm_filenames):
    # The horrible second argument here is necessary to get a V3 signature to
    # support CentOS 5 systems. See:
    # http://ilostmynotes.blogspot.com/2016/03/the-horror-of-signing-rpms-that-support.html
    args = [
        "--define=%%_gpg_name %s" % self.gpg_name,
        ("--define=__gpg_sign_cmd %%{__gpg} gpg --force-v3-sigs --yes "
         "--digest-algo=sha1 --no-verbose --no-armor --pinentry-mode loopback "
         "--no-secmem-warning -u '%s' -sbo %%{__signature_filename} "
         "%%{__plaintext_filename}" % self.gpg_name), "--addsign"
    ] + rpm_filenames

    try:
      output_log = io.StringIO()
      rpmsign = pexpect.spawn("rpmsign", args, timeout=1000)
      # Use logfile_read so we don't get the password we're injecting.
      rpmsign.logfile_read = output_log
      rpmsign.expect("phrase:")
      rpmsign.sendline(self.password)
      rpmsign.wait()
    except pexpect.exceptions.EOF:
      # This could have worked using a cached passphrase, we check for the
      # actual signature below and raise if the package wasn't signed after all.
      pass
    except pexpect.ExceptionPexpect:
      logging.error(output_log.getvalue())
      raise

    for rpm_filename in rpm_filenames:
      try:
        # Expected output is: filename.rpm: rsa sha1 (md5) pgp md5 OK
        output = subprocess.check_output(["rpm", "--checksig", rpm_filename])
        if "pgp" not in output:
          raise SigningError("PGP missing checksig %s" % rpm_filename)
      except subprocess.CalledProcessError:
        logging.exception("Bad signature verification on %s", rpm_filename)
        raise SigningError("Bad signature verification on %s" % rpm_filename)

Response: Signs a set of RPM files with the gpg signature.


Instruction:
Can you tell what is the following Python 3 function doing
def SearchClients(query_str, token=None, limit=1000):
  client_schema = aff4.AFF4Object.classes["VFSGRRClient"].SchemaCls
  index = client_index.CreateClientIndex(token=token)

  client_list = index.LookupClients([query_str])
  result_set = aff4.FACTORY.MultiOpen(client_list, token=token)
  results = []
  for result in result_set:
    results.append((result, str(result.Get(client_schema.HOSTNAME)),
                    str(result.Get(client_schema.OS_VERSION)),
                    str(result.Get(client_schema.PING))))
    if len(results) >= limit:
      break

  return results

Response: Search indexes for clients. Returns list ( client hostname os version ping ).


Instruction:
Can you tell what is the following Python 3 function doing
def DownloadDir(aff4_path, output_dir, bufsize=8192, preserve_path=True):
  if not os.path.isdir(output_dir):
    os.makedirs(output_dir)
  fd = aff4.FACTORY.Open(aff4_path)
  for child in fd.OpenChildren():
    if preserve_path:
      # Get a full path without the aff4:
      full_dir = utils.JoinPath(output_dir, child.urn.Path())
      full_dir = os.path.dirname(full_dir)
      if not os.path.isdir(full_dir):
        os.makedirs(full_dir)
      outfile = os.path.join(full_dir, child.urn.Basename())
    else:
      outfile = os.path.join(output_dir, child.urn.Basename())
    logging.info(u"Downloading %s to %s", child.urn, outfile)
    with open(outfile, "wb") as out_fd:
      try:
        buf = child.Read(bufsize)
        while buf:
          out_fd.write(buf)
          buf = child.Read(bufsize)
      except IOError as e:
        logging.error("Failed to read %s. Err: %s", child.urn, e)

Response: Download all files in a directory.


Instruction:
Can you implement a function in Python 3 that
opens the client with the given id and token.

Response:
def OpenClient(client_id=None, token=None):
  """Opens the client, getting potential approval tokens.

  Args:
    client_id: The client id that should be opened.
    token: Token to use to open the client

  Returns:
    tuple containing (client, token) objects or (None, None) on if
    no appropriate aproval tokens were found.
  """
  if not token:
    try:
      token = ApprovalFind(client_id, token=token)
    except access_control.UnauthorizedAccess as e:
      logging.debug("No authorization found for access to client: %s", e)

  try:
    # Try and open with the token we managed to retrieve or the default.
    client = aff4.FACTORY.Open(
        rdfvalue.RDFURN(client_id), mode="r", token=token)
    return client, token
  except access_control.UnauthorizedAccess:
    logging.warning(
        "Unable to find a valid reason for client %s. You may need "
        "to request approval.", client_id)
    return None, None


Instruction:
Can you tell what is the following Python 3 function doing
def GetNotifications(user=None, token=None):
  if not user:
    user = getpass.getuser()
  user_obj = aff4.FACTORY.Open(
      aff4.ROOT_URN.Add("users").Add(user), token=token)
  return list(user_obj.Get(user_obj.Schema.PENDING_NOTIFICATIONS))

Response: Show pending notifications for a user.


Instruction:
Can you tell what is the following Python 3 function doing
def ApprovalGrant(token=None):
  user = getpass.getuser()
  notifications = GetNotifications(user=user, token=token)
  requests = [n for n in notifications if n.type == "GrantAccess"]
  for request in requests:
    _, client_id, user, reason = rdfvalue.RDFURN(request.subject).Split()
    reason = utils.DecodeReasonString(reason)
    print(request)
    print("Reason: %s" % reason)
    if input("Do you approve this request? [y/N] ").lower() == "y":
      security.ClientApprovalGrantor(
          subject_urn=client_id, reason=reason, delegate=user,
          token=token).Grant()
      # TODO(user): Remove the notification.
    else:
      print("skipping request")
    print("Approval sent")

Response: Iterate through requested access approving or not.


Instruction:
Can you implement a function in Python 3 that
finds approvals issued for a specific client.

Response:
def ApprovalFind(object_id, token=None):
  """Find approvals issued for a specific client."""
  user = getpass.getuser()
  object_id = rdfvalue.RDFURN(object_id)
  try:
    approved_token = security.Approval.GetApprovalForObject(
        object_id, token=token, username=user)
    print("Found token %s" % str(approved_token))
    return approved_token
  except access_control.UnauthorizedAccess:
    print("No token available for access to %s" % object_id)


Instruction:
Write a Python 3 script for
creating an approval with raw datastore access.

Response:
def ApprovalCreateRaw(aff4_path,
                      reason="",
                      expire_in=60 * 60 * 24 * 7,
                      token=None,
                      approval_type="ClientApproval"):
  """Creates an approval with raw access.

  This method requires raw datastore access to manipulate approvals directly.
  This currently doesn't work for hunt or cron approvals, because they check
  that each approver has the admin label.  Since the fake users don't exist the
  check fails.

  Args:
    aff4_path: The aff4_path or client id the approval should be created for.
    reason: The reason to put in the token.
    expire_in: Expiry in seconds to use in the token.
    token: The token that will be used. If this is specified reason and expiry
      are ignored.
    approval_type: The type of the approval to create.

  Returns:
    The token.

  Raises:
    RuntimeError: On bad token.
  """
  if approval_type in ["ClientApproval", security.ClientApproval]:
    urn = rdf_client.ClientURN(aff4_path)
  else:
    urn = rdfvalue.RDFURN(aff4_path)

  if not token:
    expiry = time.time() + expire_in
    token = access_control.ACLToken(reason=reason, expiry=expiry)

  if not token.reason:
    raise RuntimeError("Cannot create approval with empty reason")
  if not token.username:
    token.username = getpass.getuser()
  approval_urn = security.ApprovalRequestor.ApprovalUrnBuilder(
      urn.Path(), token.username, token.reason)
  super_token = access_control.ACLToken(username="raw-approval-superuser")
  super_token.supervisor = True

  if isinstance(approval_type, string_types):
    approval_type_cls = aff4.AFF4Object.classes[approval_type]
  else:
    approval_type_cls = approval_type

  approval_request = aff4.FACTORY.Create(
      approval_urn, approval_type_cls, mode="rw", token=super_token)

  # Add approvals indicating they were approved by fake "raw" mode users.
  approval_request.AddAttribute(
      approval_request.Schema.APPROVER("%s1-raw" % token.username))
  approval_request.AddAttribute(
      approval_request.Schema.APPROVER("%s-raw2" % token.username))
  approval_request.Close()


Instruction:
Given the following Python 3 function, write the documentation
def ApprovalRevokeRaw(aff4_path, token):
  try:
    urn = rdf_client.ClientURN(aff4_path)
  except type_info.TypeValueError:
    urn = rdfvalue.RDFURN(aff4_path)

  approval_urn = aff4.ROOT_URN.Add("ACL").Add(urn.Path()).Add(
      token.username).Add(utils.EncodeReasonString(token.reason))

  super_token = access_control.ACLToken(username="raw-approval-superuser")
  super_token.supervisor = True

  approval_request = aff4.FACTORY.Open(
      approval_urn, mode="rw", token=super_token)
  approval_request.DeleteAttribute(approval_request.Schema.APPROVER)
  approval_request.Close()

Response: Revokes an approval for a given token.


Instruction:
Implement a Python 3 function for
opening the given clients in batches and returns hardware information.

Response:
def _GetHWInfos(client_list, batch_size=10000, token=None):
  """Opens the given clients in batches and returns hardware information."""

  # This function returns a dict mapping each client_id to a set of reported
  # hardware serial numbers reported by this client.
  hw_infos = {}

  logging.info("%d clients to process.", len(client_list))

  c = 0

  for batch in collection.Batch(client_list, batch_size):
    logging.info("Processing batch: %d-%d", c, c + batch_size)
    c += len(batch)

    client_objs = aff4.FACTORY.MultiOpen(batch, age=aff4.ALL_TIMES, token=token)

    for client in client_objs:
      hwi = client.GetValuesForAttribute(client.Schema.HARDWARE_INFO)

      hw_infos[client.urn] = set(["%s" % x.serial_number for x in hwi])

  return hw_infos


Instruction:
Explain what the following Python 3 code does
def CleanClientVersions(clients=None, dry_run=True, token=None):
  if not clients:
    index = client_index.CreateClientIndex(token=token)
    clients = index.LookupClients(["."])
  clients.sort()
  with data_store.DB.GetMutationPool() as pool:

    logging.info("checking %d clients", len(clients))

    # TODO(amoser): This only works on datastores that use the Bigtable scheme.
    client_infos = data_store.DB.MultiResolvePrefix(
        clients, "aff4:type", data_store.DB.ALL_TIMESTAMPS)

    for client, type_list in client_infos:
      logging.info("%s: has %d versions", client, len(type_list))
      cleared = 0
      kept = 1
      last_kept = type_list[0][2]
      for _, _, ts in type_list[1:]:
        if last_kept - ts > 60 * 60 * 1000000:  # 1 hour
          last_kept = ts
          kept += 1
        else:
          if not dry_run:
            pool.DeleteAttributes(client, ["aff4:type"], start=ts, end=ts)
          cleared += 1
          if pool.Size() > 10000:
            pool.Flush()
      logging.info("%s: kept %d and cleared %d", client, kept, cleared)

Response: This function cleans up excessive client versions.


Instruction:
Write a Python 3 function that can
start a flow and a worker.

Response:
def StartFlowAndWorker(client_id, flow_name, **kwargs):
  """Launches the flow and worker and waits for it to finish.

  Args:
     client_id: The client common name we issue the request.
     flow_name: The name of the flow to launch.
     **kwargs: passthrough to flow.

  Returns:
     A flow session id.

  Note: you need raw access to run this flow as it requires running a worker.
  """
  # Empty token, only works with raw access.
  queue = rdfvalue.RDFURN("DEBUG-%s-" % getpass.getuser())
  if "token" in kwargs:
    token = kwargs.pop("token")
  else:
    token = access_control.ACLToken(username="GRRConsole")

  session_id = flow.StartAFF4Flow(
      client_id=client_id,
      flow_name=flow_name,
      queue=queue,
      token=token,
      **kwargs)
  worker_thrd = worker_lib.GRRWorker(
      queues=[queue], token=token, threadpool_size=1)
  while True:
    try:
      worker_thrd.RunOnce()
    except KeyboardInterrupt:
      print("exiting")
      worker_thrd.thread_pool.Join()
      break

    time.sleep(2)
    with aff4.FACTORY.Open(session_id, token=token) as flow_obj:
      if not flow_obj.GetRunner().IsRunning():
        break

  # Terminate the worker threads
  worker_thrd.thread_pool.Join()

  return session_id


Instruction:
Implement a Python 3 function for
waking up stuck flows.

Response:
def WakeStuckFlow(session_id):
  """Wake up stuck flows.

  A stuck flow is one which is waiting for the client to do something, but the
  client requests have been removed from the client queue. This can happen if
  the system is too loaded and the client messages have TTLed out. In this case
  we reschedule the client requests for this session.

  Args:
    session_id: The session for the flow to wake.

  Returns:
    The total number of client messages re-queued.
  """
  session_id = rdfvalue.SessionID(session_id)
  woken = 0
  checked_pending = False

  with queue_manager.QueueManager() as manager:
    for request, responses in manager.FetchRequestsAndResponses(session_id):
      # We need to check if there are client requests pending.
      if not checked_pending:
        task = manager.Query(
            request.client_id, task_id="task:%s" % request.request.task_id)

        if task:
          # Client has tasks pending already.
          return

        checked_pending = True

      if (not responses or
          responses[-1].type != rdf_flows.GrrMessage.Type.STATUS):
        manager.QueueClientMessage(request.request)
        woken += 1

      if responses and responses[-1].type == rdf_flows.GrrMessage.Type.STATUS:
        manager.QueueNotification(session_id)

  return woken


Instruction:
Can you tell what is the following Python 3 function doing
def RunOnce(self):

    # Figure out which port to use.
    port = config.CONFIG["Monitoring.http_port"]
    if not port:
      logging.info("Monitoring server disabled.")
      return

    # TODO(user): Implement __contains__ for GrrConfigManager.
    max_port = config.CONFIG.Get("Monitoring.http_port_max", None)
    if max_port is None:
      # Use the same number of available ports as the adminui is using. If we
      # have 10 available for adminui we will need 10 for the stats server.
      adminui_max_port = config.CONFIG.Get("AdminUI.port_max",
                                           config.CONFIG["AdminUI.port"])
      max_port = port + adminui_max_port - config.CONFIG["AdminUI.port"]

    try:
      # pylint: disable=g-import-not-at-top
      from grr_response_server.local import stats_server
      # pylint: enable=g-import-not-at-top
      server_cls = stats_server.StatsServer
      logging.debug("Using local StatsServer")
    except ImportError:
      logging.debug("Using default StatsServer")
      server_cls = StatsServer

    for port in range(port, max_port + 1):
      try:
        logging.info("Starting monitoring server on port %d.", port)
        server_obj = server_cls(port)
        server_obj.Start()
        return
      except base_stats_server.PortInUseError as e:
        if e.port < max_port:
          logging.info(e)
          continue
        raise

Response: This method is called by the base class to start monitoring server.


Instruction:
How would you explain what the following Python 3 function does
def Parse(self, stat, file_object, knowledge_base):
    _, _ = stat, knowledge_base

    packages = []
    sw_data = utils.ReadFileBytesAsUnicode(file_object)
    try:
      for pkg in self._deb822.Packages.iter_paragraphs(sw_data.splitlines()):
        if self.installed_re.match(pkg["Status"]):
          packages.append(
              rdf_client.SoftwarePackage(
                  name=pkg["Package"],
                  description=pkg["Description"],
                  version=pkg["Version"],
                  architecture=pkg["Architecture"],
                  publisher=pkg["Maintainer"],
                  install_state="INSTALLED"))
    except SystemError:
      yield rdf_anomaly.Anomaly(
          type="PARSER_ANOMALY", symptom="Invalid dpkg status file")
    finally:
      if packages:
        yield rdf_client.SoftwarePackages(packages=packages)

Response: Parse the status file.


Instruction:
Here you have a function in Python 3, explain what it does
def Is64bit(self):
    if "64" not in platform.machine():
      return False
    iswow64 = ctypes.c_bool(False)
    if IsWow64Process is None:
      return False
    if not IsWow64Process(self.h_process, ctypes.byref(iswow64)):
      raise process_error.ProcessError("Error while calling IsWow64Process.")
    return not iswow64.value

Response: Returns true if this is a 64 bit process.


Instruction:
Create a Python 3 function for
opening the process for reading.

Response:
def Open(self):
    """Opens the process for reading."""

    self.h_process = kernel32.OpenProcess(
        PROCESS_VM_READ | PROCESS_QUERY_INFORMATION, 0, self.pid)
    if not self.h_process:
      raise process_error.ProcessError(
          "Failed to open process (pid %d)." % self.pid)

    if self.Is64bit():
      si = self.GetNativeSystemInfo()
      self.max_addr = si.lpMaximumApplicationAddress
    else:
      si = self.GetSystemInfo()
      self.max_addr = 2147418111

    self.min_addr = si.lpMinimumApplicationAddress


Instruction:
How would you implement a function in Python 3 that
reads at most num_bytes starting from offset <address >.

Response:
def ReadBytes(self, address, num_bytes):
    """Reads at most num_bytes starting from offset <address>."""
    address = int(address)
    buf = ctypes.create_string_buffer(num_bytes)
    bytesread = ctypes.c_size_t(0)
    res = ReadProcessMemory(self.h_process, address, buf, num_bytes,
                            ctypes.byref(bytesread))
    if res == 0:
      err = wintypes.GetLastError()
      if err == 299:
        # Only part of ReadProcessMemory has been done, let's return it.
        return buf.raw[:bytesread.value]
      raise process_error.ProcessError("Error in ReadProcessMemory: %d" % err)

    return buf.raw[:bytesread.value]


Instruction:
How would you explain what the following Python 3 function does
def _MigrationFilenameToInt(fname):
  base, _ = os.path.splitext(fname)
  return int(base)

Response: Converts a migration filename to a migration number.


Instruction:
Create a Python 3 function for
listing filenames of migrations with numbers bigger than a given one.

Response:
def ListMigrationsToProcess(migrations_root,
                            current_migration_number
                           ):
  """Lists filenames of migrations with numbers bigger than a given one."""
  migrations = []
  for m in os.listdir(migrations_root):
    if (current_migration_number is None or
        _MigrationFilenameToInt(m) > current_migration_number):
      migrations.append(m)

  return sorted(migrations, key=_MigrationFilenameToInt)


Instruction:
Given the following Python 3 function, write the documentation
def ProcessMigrations(open_conn_fn,
                      migrations_root):
  """Processes migrations from a given folder.

  This function uses LOCK TABLE MySQL command on _migrations
  table to ensure that only one GRR process is actually
  performing the migration.

  We have to use open_conn_fn to open 2 connections to the database,
  since LOCK TABLE command is per-connection and it's not allowed
  to modify non-locked tables if LOCK TABLE was called within a
  connection. To overcome this limitation we use one connection
  to lock _migrations and perform its updates and one connection
  to do actual migrations.

  Args:
    open_conn_fn: A function to open new database connection.
    migrations_root: A path to folder with migration files.
  """
  with contextlib.closing(open_conn_fn()) as conn:
    conn.autocommit(True)

    with contextlib.closing(conn.cursor()) as cursor:
      cursor.execute("""CREATE TABLE IF NOT EXISTS _migrations(
        migration_id INT UNSIGNED PRIMARY KEY,
        timestamp TIMESTAMP(6) NOT NULL DEFAULT NOW(6)
        )""")

    with contextlib.closing(conn.cursor()) as cursor:
      cursor.execute('SELECT GET_LOCK("grr_migration", 3600)')

    try:
      with contextlib.closing(conn.cursor()) as cursor:
        current_migration = GetLatestMigrationNumber(cursor)

      to_process = ListMigrationsToProcess(migrations_root, current_migration)
      logging.info("Will execute following DB migrations: %s",
                   ", ".join(to_process))

      for fname in to_process:
        start_time = time.time()
        logging.info("Starting migration %s", fname)

        with open(os.path.join(migrations_root, fname)) as fd:
          sql = fd.read()
          with contextlib.closing(conn.cursor()) as cursor:
            cursor.execute(sql)

        logging.info("Migration %s is done. Took %.2fs", fname,
                     time.time() - start_time)

        # Update _migrations table with the latest migration.
        with contextlib.closing(conn.cursor()) as cursor:
          cursor.execute("INSERT INTO _migrations (migration_id) VALUES (%s)",
                         [_MigrationFilenameToInt(fname)])
    finally:
      with contextlib.closing(conn.cursor()) as cursor:
        cursor.execute('SELECT RELEASE_LOCK("grr_migration")')

Response: Processes migrations from a given folder.


Instruction:
Create a Python 3 function to
dump the current database schema.

Response:
def DumpCurrentSchema(cursor):
  """Dumps current database schema."""
  cursor.execute("SELECT TABLE_NAME FROM INFORMATION_SCHEMA.TABLES "
                 "WHERE table_schema = (SELECT DATABASE())")
  defs = []
  for table, in sorted(cursor.fetchall()):
    cursor.execute("SHOW CREATE TABLE `{}`".format(table))
    rows = cursor.fetchall()
    defs.append(rows[0][1])

  return "\n\n".join(defs)


Instruction:
Can you implement a function in Python 3 that
filters a list by filtering elements starting at offset.

Response:
def FilterList(l, offset, count=0, filter_value=None):
  """Filters a list, getting count elements, starting at offset."""

  if offset < 0:
    raise ValueError("Offset needs to be greater than or equal to zero")

  if count < 0:
    raise ValueError("Count needs to be greater than or equal to zero")

  count = count or sys.maxsize
  if not filter_value:
    return l[offset:offset + count]

  index = 0
  items = []
  for item in l:
    serialized_item = item.SerializeToString()
    if re.search(re.escape(filter_value), serialized_item, re.I):
      if index >= offset:
        items.append(item)
      index += 1

      if len(items) >= count:
        break

  return items


Instruction:
Can you generate the documentation for the following Python 3 function
def FilterCollection(aff4_collection, offset, count=0, filter_value=None):

  if offset < 0:
    raise ValueError("Offset needs to be greater than or equal to zero")

  if count < 0:
    raise ValueError("Count needs to be greater than or equal to zero")

  count = count or sys.maxsize
  if filter_value:
    index = 0
    items = []
    for item in aff4_collection.GenerateItems():
      serialized_item = item.SerializeToString()
      if re.search(re.escape(filter_value), serialized_item, re.I):
        if index >= offset:
          items.append(item)
        index += 1

        if len(items) >= count:
          break
  else:
    items = list(itertools.islice(aff4_collection.GenerateItems(offset), count))

  return items

Response: Filters an aff4 collection starting at offset.


Instruction:
Can you generate the documentation for the following Python 3 function
def get_config():
  # TODO(hanuszczak): See comment in `setup.py` for `grr-response-proto`.
  ini_path = os.path.join(THIS_DIRECTORY, "version.ini")
  if not os.path.exists(ini_path):
    ini_path = os.path.join(THIS_DIRECTORY, "../../version.ini")
    if not os.path.exists(ini_path):
      raise RuntimeError("Couldn't find version.ini")

  config = configparser.SafeConfigParser()
  config.read(ini_path)
  return config

Response: Get INI parser with version. ini data.


Instruction:
Make a summary of the following Python 3 code
def CopyFiles(self):
    if self.fleetspeak_enabled:
      # Copy files needed for dpkg-buildpackage.
      shutil.copytree(
          config_lib.Resource().Filter(
              "install_data/debian/dpkg_client/fleetspeak-debian"),
          os.path.join(self.package_dir, "debian/debian.in"))
      return

    # Copy the nanny binary.
    shutil.copy(
        package.ResourcePath("grr-response-core",
                             "install_data/debian/dpkg_client/nanny.sh.in"),
        self.output_dir)

    # Copy files needed for dpkg-buildpackage.
    shutil.copytree(
        config_lib.Resource().Filter("install_data/debian/dpkg_client/debian"),
        os.path.join(self.package_dir, "debian/debian.in"))

    # Copy upstart files
    outdir = os.path.join(self.package_dir, "debian/upstart.in")
    utils.EnsureDirExists(outdir)
    shutil.copy(
        config_lib.Resource().Filter(
            "install_data/debian/dpkg_client/upstart/grr-client.conf"), outdir)

    # Copy init files
    outdir = os.path.join(self.package_dir, "debian/initd.in")
    utils.EnsureDirExists(outdir)
    shutil.copy(
        config_lib.Resource().Filter(
            "install_data/debian/dpkg_client/initd/grr-client"), outdir)

    # Copy systemd unit file
    outdir = os.path.join(self.package_dir, "debian/systemd.in")
    utils.EnsureDirExists(outdir)
    shutil.copy(
        config_lib.Resource().Filter(
            "install_data/systemd/client/grr-client.service"), outdir)

Response: This copies the files needed for the debian and nanny binary.


Instruction:
Write a Python 3 script to
create a ZIP archive of the files in the input directory.

Response:
def MakeZip(self, input_dir, output_file):
    """Creates a ZIP archive of the files in the input directory.

    Args:
      input_dir: the name of the input directory.
      output_file: the name of the output ZIP archive without extension.
    """

    logging.info("Generating zip template file at %s", output_file)
    zf = zipfile.ZipFile(output_file, "w")
    oldwd = os.getcwd()
    os.chdir(input_dir)
    for path in ["debian", "rpmbuild", "fleetspeak"]:
      for root, _, files in os.walk(path):
        for f in files:
          zf.write(os.path.join(root, f))
    zf.close()
    os.chdir(oldwd)


Instruction:
Can you tell what is the following Python 3 function doing
def CopyFiles(self):

    shutil.move(
        os.path.join(self.package_dir, "debian"),
        os.path.join(self.package_dir, "rpmbuild"))
    if self.fleetspeak_enabled:
      rpm_spec_filename = "fleetspeak.grr.spec.in"
    else:
      rpm_spec_filename = "grr.spec.in"
      shutil.copy(
          config_lib.Resource().Filter(
              "install_data/centos/grr-client.initd.in"),
          os.path.join(self.package_dir, "rpmbuild/grr-client.initd.in"))
      shutil.copy(
          config_lib.Resource().Filter(
              "install_data/systemd/client/grr-client.service"),
          os.path.join(self.package_dir, "rpmbuild/grr-client.service.in"))

    shutil.copy(
        config_lib.Resource().Filter("install_data/centos/" +
                                     rpm_spec_filename),
        os.path.join(self.package_dir, "rpmbuild/grr.spec.in"))

    # Copy prelink blacklist file. Without this file, prelink will mangle
    # the GRR binary.
    shutil.copy(
        config_lib.Resource().Filter(
            "install_data/centos/prelink_blacklist.conf.in"),
        os.path.join(self.package_dir, "rpmbuild/prelink_blacklist.conf.in"))

Response: This copies the files from the build directory to the build directory.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def SecurityCheck(func):

  def Wrapper(request, *args, **kwargs):
    """Wrapping function."""
    if WEBAUTH_MANAGER is None:
      raise RuntimeError("Attempt to initialize before WEBAUTH_MANAGER set.")
    return WEBAUTH_MANAGER.SecurityCheck(func, request, *args, **kwargs)

  return Wrapper

Response: A decorator applied to protected web handlers.


Instruction:
Write a Python 3 function that can
check if access should be allowed for the request.

Response:
def SecurityCheck(self, func, request, *args, **kwargs):
    """Check if access should be allowed for the request."""

    try:
      auth_header = request.headers.get("Authorization", "")
      if not auth_header.startswith(self.BEARER_PREFIX):
        raise ValueError("JWT token is missing.")

      token = auth_header[len(self.BEARER_PREFIX):]

      auth_domain = config.CONFIG["AdminUI.firebase_auth_domain"]
      project_id = auth_domain.split(".")[0]

      idinfo = id_token.verify_firebase_token(
          token, request, audience=project_id)

      if idinfo["iss"] != self.SECURE_TOKEN_PREFIX + project_id:
        raise ValueError("Wrong issuer.")

      request.user = idinfo["email"]
    except ValueError as e:
      # For a homepage, just do a pass-through, otherwise JS code responsible
      # for the Firebase auth won't ever get executed. This approach is safe,
      # because wsgiapp.HttpRequest object will raise on any attempt to
      # access uninitialized HttpRequest.user attribute.
      if request.path != "/":
        return self.AuthError("JWT token validation failed: %s" % e)

    return func(request, *args, **kwargs)


Instruction:
Write a Python 3 script to
run this once on init.

Response:
def RunOnce(self):
    """Run this once on init."""
    global WEBAUTH_MANAGER  # pylint: disable=global-statement

    # pylint: disable=g-bad-name
    WEBAUTH_MANAGER = BaseWebAuthManager.GetPlugin(
        config.CONFIG["AdminUI.webauth_manager"])()

    # pylint: enable=g-bad-name
    logging.info("Using webauth manager %s", WEBAUTH_MANAGER)


Instruction:
Can you create a Python 3 function that
returns a dictionary of annotated router methods.

Response:
def GetAnnotatedMethods(cls):
    """Returns a dictionary of annotated router methods."""

    result = {}

    # We want methods with the highest call-order to be processed last,
    # so that their annotations have precedence.
    for i_cls in reversed(inspect.getmro(cls)):
      for name in compatibility.ListAttrs(i_cls):
        cls_method = getattr(i_cls, name)

        if not callable(cls_method):
          continue

        if not hasattr(cls_method, "__http_methods__"):
          continue

        result[name] = RouterMethodMetadata(
            name=name,
            doc=cls_method.__doc__,
            args_type=getattr(cls_method, "__args_type__", None),
            result_type=getattr(cls_method, "__result_type__", None),
            category=getattr(cls_method, "__category__", None),
            http_methods=getattr(cls_method, "__http_methods__", set()),
            no_audit_log_required=getattr(cls_method,
                                          "__no_audit_log_required__", False))

    return result


Instruction:
Can you implement a function in Python 3 that
publishes a message into all listeners of the event.

Response:
def PublishEvent(cls, event_name, msg, token=None):
    """Publish the message into all listeners of the event.

    We send the message to all event handlers which contain this
    string in their EVENT static member. This allows the event to be
    sent to multiple interested listeners.

    Args:
      event_name: An event name.
      msg: The message to send to the event handler.
      token: ACL token.

    Raises:
      ValueError: If the message is invalid. The message must be a Semantic
        Value (instance of RDFValue) or a full GrrMessage.
    """
    cls.PublishMultipleEvents({event_name: [msg]}, token=token)


Instruction:
Can you write a function in Python 3 where it
publishes multiple messages at once.

Response:
def PublishMultipleEvents(cls, events, token=None):
    """Publishes multiple messages at once.

    Args:
      events: A dict with keys being event names and values being lists of
        messages.
      token: ACL token.

    Raises:
      ValueError: If the message is invalid. The message must be a Semantic
        Value (instance of RDFValue) or a full GrrMessage.
    """
    event_name_map = registry.EventRegistry.EVENT_NAME_MAP
    for event_name, messages in iteritems(events):
      if not isinstance(event_name, string_types):
        raise ValueError(
            "Event names should be string, got: %s" % type(event_name))
      for msg in messages:
        if not isinstance(msg, rdfvalue.RDFValue):
          raise ValueError("Can only publish RDFValue instances.")

      for event_cls in event_name_map.get(event_name, []):
        event_cls().ProcessMessages(messages, token=token)


Instruction:
How would you implement a function in Python 3 that
overlays a child hint with a parent hint.

Response:
def Overlay(child, parent):
  """Adds hint attributes to a child hint if they are not defined."""
  for arg in child, parent:
    if not isinstance(arg, collections.Mapping):
      raise DefinitionError("Trying to merge badly defined hints. Child: %s, "
                            "Parent: %s" % (type(child), type(parent)))
  for attr in ["fix", "format", "problem", "summary"]:
    if not child.get(attr):
      child[attr] = parent.get(attr, "").strip()
  return child


Instruction:
Can you generate a brief explanation for the following Python 3 code
def FanOut(self, obj, parent=None):
    # Catch cases where RDFs are iterable but return themselves.
    if parent and obj == parent:
      results = [utils.SmartUnicode(obj).strip()]
    elif isinstance(obj, (string_types, rdf_structs.EnumNamedValue)):
      results = [utils.SmartUnicode(obj).strip()]
    elif isinstance(obj, rdf_protodict.DataBlob):
      results = self.FanOut(obj.GetValue())
    elif isinstance(obj, (collections.Mapping, rdf_protodict.Dict)):
      results = []
      # rdf_protodict.Dict only has items, not iteritems.
      for k, v in iteritems(obj):
        expanded_v = [utils.SmartUnicode(r) for r in self.FanOut(v)]
        results.append("%s:%s" % (utils.SmartUnicode(k), ",".join(expanded_v)))
    elif isinstance(obj, (collections.Iterable,
                          rdf_structs.RepeatedFieldHelper)):
      results = []
      for rslt in [self.FanOut(o, obj) for o in obj]:
        results.extend(rslt)
    else:
      results = [utils.SmartUnicode(obj).strip()]
    return results

Response: Expand values from various attribute types.


Instruction:
How would you explain what the following Python 3 function does
def Format(self, format_string, rdf):
    result = []
    for literal_text, field_name, _, _ in self.parse(format_string):
      # output the literal text
      if literal_text:
        result.append(literal_text)
      # if there's a field, output it
      if field_name is not None:
        rslts = []
        objs = self.expander(rdf, field_name)
        for o in objs:
          rslts.extend(self.FanOut(o))
        # format the objects and append to the result
        result.append(",".join(rslts))
    return "".join(result)

Response: Apply string formatting templates to rdf data.


Instruction:
How would you implement a function in Python 3 that
refreshes an attribute of an object.

Response:
def Update(self, attribute=None):
    """Refresh an old attribute.

    Note that refreshing the attribute is asynchronous. It does not change
    anything about the current object - you need to reopen the same URN some
    time later to get fresh data.

    Attributes: CONTAINS - Refresh the content of the directory listing.
    Args:
       attribute: An attribute object as listed above.

    Returns:
       The Flow ID that is pending

    Raises:
       IOError: If there has been an error starting the flow.
    """
    # client id is the first path element
    client_id = self.urn.Split()[0]

    if attribute == "CONTAINS":
      # Get the pathspec for this object
      flow_id = flow.StartAFF4Flow(
          client_id=client_id,
          # Dependency loop: aff4_objects/aff4_grr.py depends on
          # aff4_objects/standard.py that depends on flows/general/filesystem.py
          # that eventually depends on aff4_objects/aff4_grr.py
          # flow_name=filesystem.ListDirectory.__name__,
          flow_name="ListDirectory",
          pathspec=self.real_pathspec,
          notify_to_user=False,
          token=self.token)

      return flow_id


Instruction:
Can you generate the documentation for the following Python 3 function
def _GetChunkForReading(self, chunk):
    try:
      return self.chunk_cache.Get(chunk)
    except KeyError:
      pass

    # We don't have this chunk already cached. The most common read
    # access pattern is contiguous reading so since we have to go to
    # the data store already, we read ahead to reduce round trips.

    missing_chunks = []
    for chunk_number in range(chunk, chunk + 10):
      if chunk_number not in self.chunk_cache:
        missing_chunks.append(chunk_number)

    self._ReadChunks(missing_chunks)
    # This should work now - otherwise we just give up.
    try:
      return self.chunk_cache.Get(chunk)
    except KeyError:
      raise aff4.ChunkNotFoundError("Cannot open chunk %s" % chunk)

Response: Returns the relevant chunk from the datastore and reads ahead.


Instruction:
Write a Python 3 script to
return the relevant chunk from the datastore.

Response:
def _GetChunkForWriting(self, chunk):
    """Returns the relevant chunk from the datastore."""
    try:
      chunk = self.chunk_cache.Get(chunk)
      chunk.dirty = True
      return chunk
    except KeyError:
      pass

    try:
      chunk = self._ReadChunk(chunk)
      chunk.dirty = True
      return chunk
    except KeyError:
      pass

    fd = io.BytesIO()
    fd.chunk = chunk
    fd.dirty = True
    self.chunk_cache.Put(chunk, fd)

    # Keep track of the biggest chunk_number we've seen so far.
    if chunk > self.last_chunk:
      self.last_chunk = chunk
      self._dirty = True

    return fd


Instruction:
Can you tell what is the following Python 3 function doing
def _ReadPartial(self, length):
    chunk = self.offset // self.chunksize
    chunk_offset = self.offset % self.chunksize

    # If we're past the end of the file, we don't have a chunk to read from, so
    # we can't read anymore. We return the empty string here so we can read off
    # the end of a file without raising, and get as much data as is there.
    if chunk > self.last_chunk:
      return ""

    available_to_read = min(length, self.chunksize - chunk_offset)

    fd = self._GetChunkForReading(chunk)

    fd.seek(chunk_offset)

    result = fd.read(available_to_read)
    self.offset += len(result)

    return result

Response: Read as much as possible but not more than length.


Instruction:
How would you explain what the following Python 3 function does
def AddBlob(self, blob_hash, length, chunk_number):
    if len(blob_hash.AsBytes()) != self._HASH_SIZE:
      raise ValueError("Hash '%s' doesn't have correct length (%d)." %
                       (blob_hash, self._HASH_SIZE))

    # If we're adding a new blob, we should increase the size. If we're just
    # updating an existing blob, the size should stay the same.
    # That is, if we read the index at the right offset and no hash is there, we
    # must not have seen this blob before, so we say we're adding a new one and
    # increase in size.
    if not self.ChunkExists(chunk_number):
      # We say that we've increased in size by the size of the blob,
      # but really we only store its hash in the AFF4SparseImage.
      self.size += length
      self._dirty = True
      # Keep track of the biggest chunk_number we've seen so far.
      if chunk_number > self.last_chunk:
        self.last_chunk = chunk_number
        self._dirty = True

    index_urn = self.urn.Add(self.CHUNK_ID_TEMPLATE % chunk_number)
    # TODO(amoser): This opens a subobject for each AddBlob call :/
    with aff4.FACTORY.Create(
        index_urn, aff4.AFF4MemoryStream, token=self.token) as fd:
      fd.write(blob_hash.AsBytes())
    if chunk_number in self.chunk_cache:
      self.chunk_cache.Pop(chunk_number)

Response: Adds a blob to this image using its hash and length.


Instruction:
Create a Python 3 function for
checking if the given chunk numbers exist in the index.

Response:
def ChunksExist(self, chunk_numbers):
    """Do we have this chunk in the index?"""
    index_urns = {
        self.urn.Add(self.CHUNK_ID_TEMPLATE % chunk_number): chunk_number
        for chunk_number in chunk_numbers
    }

    res = {chunk_number: False for chunk_number in chunk_numbers}

    for metadata in aff4.FACTORY.Stat(index_urns):
      res[index_urns[metadata["urn"]]] = True

    return res


Instruction:
Can you tell what is the following Python 3 function doing
def Flush(self):
    super(LabelSet, self).Flush()

    self.to_delete = self.to_delete.difference(self.to_set)

    with data_store.DB.GetMutationPool() as mutation_pool:
      mutation_pool.LabelUpdateLabels(
          self.urn, self.to_set, to_delete=self.to_delete)
    self.to_set = set()
    self.to_delete = set()

Response: Flush the data to the index.


Instruction:
Can you generate the documentation for the following Python 3 function
def WriteCronJob(self, cronjob, cursor=None):
    query = ("INSERT INTO cron_jobs "
             "(job_id, job, create_time, enabled) "
             "VALUES (%s, %s, FROM_UNIXTIME(%s), %s) "
             "ON DUPLICATE KEY UPDATE "
             "enabled=VALUES(enabled)")

    create_time_str = mysql_utils.RDFDatetimeToTimestamp(
        cronjob.created_at or rdfvalue.RDFDatetime.Now())
    cursor.execute(query, [
        cronjob.cron_job_id,
        cronjob.SerializeToString(), create_time_str, cronjob.enabled
    ])

Response: Writes a cronjob to the database.


Instruction:
Here you have a function in Python 3, explain what it does
def _CronJobFromRow(self, row):
    (job, create_time, enabled, forced_run_requested, last_run_status,
     last_run_time, current_run_id, state, leased_until, leased_by) = row

    job = rdf_cronjobs.CronJob.FromSerializedString(job)
    job.current_run_id = db_utils.IntToCronJobRunID(current_run_id)
    job.enabled = enabled
    job.forced_run_requested = forced_run_requested
    job.last_run_status = last_run_status
    job.last_run_time = mysql_utils.TimestampToRDFDatetime(last_run_time)
    if state:
      job.state = rdf_protodict.AttributedDict.FromSerializedString(state)
    job.created_at = mysql_utils.TimestampToRDFDatetime(create_time)
    job.leased_until = mysql_utils.TimestampToRDFDatetime(leased_until)
    job.leased_by = leased_by
    return job

Response: Creates a cronjob object from a database result row.


Instruction:
Explain what the following Python 3 code does
def ReadCronJobs(self, cronjob_ids=None, cursor=None):
    query = ("SELECT job, UNIX_TIMESTAMP(create_time), enabled, "
             "forced_run_requested, last_run_status, "
             "UNIX_TIMESTAMP(last_run_time), current_run_id, state, "
             "UNIX_TIMESTAMP(leased_until), leased_by "
             "FROM cron_jobs")
    if cronjob_ids is None:
      cursor.execute(query)
      return [self._CronJobFromRow(row) for row in cursor.fetchall()]

    query += " WHERE job_id IN (%s)" % ", ".join(["%s"] * len(cronjob_ids))
    cursor.execute(query, cronjob_ids)
    res = []
    for row in cursor.fetchall():
      res.append(self._CronJobFromRow(row))

    if len(res) != len(cronjob_ids):
      missing = set(cronjob_ids) - set([c.cron_job_id for c in res])
      raise db.UnknownCronJobError("CronJob(s) with id(s) %s not found." %
                                   missing)
    return res

Response: Reads all cronjobs from the database.


Instruction:
Can you implement a function in Python 3 that
updates the information for an existing cron job.

Response:
def UpdateCronJob(self,
                    cronjob_id,
                    last_run_status=db.Database.unchanged,
                    last_run_time=db.Database.unchanged,
                    current_run_id=db.Database.unchanged,
                    state=db.Database.unchanged,
                    forced_run_requested=db.Database.unchanged,
                    cursor=None):
    """Updates run information for an existing cron job."""
    updates = []
    args = []
    if last_run_status != db.Database.unchanged:
      updates.append("last_run_status=%s")
      args.append(int(last_run_status))
    if last_run_time != db.Database.unchanged:
      updates.append("last_run_time=FROM_UNIXTIME(%s)")
      args.append(mysql_utils.RDFDatetimeToTimestamp(last_run_time))
    if current_run_id != db.Database.unchanged:
      updates.append("current_run_id=%s")
      args.append(db_utils.CronJobRunIDToInt(current_run_id))
    if state != db.Database.unchanged:
      updates.append("state=%s")
      args.append(state.SerializeToString())
    if forced_run_requested != db.Database.unchanged:
      updates.append("forced_run_requested=%s")
      args.append(forced_run_requested)

    if not updates:
      return

    query = "UPDATE cron_jobs SET "
    query += ", ".join(updates)
    query += " WHERE job_id=%s"
    res = cursor.execute(query, args + [cronjob_id])
    if res != 1:
      raise db.UnknownCronJobError("CronJob with id %s not found." % cronjob_id)


Instruction:
How would you explain what the following Python 3 function does
def LeaseCronJobs(self, cronjob_ids=None, lease_time=None, cursor=None):
    now = rdfvalue.RDFDatetime.Now()
    now_str = mysql_utils.RDFDatetimeToTimestamp(now)
    expiry_str = mysql_utils.RDFDatetimeToTimestamp(now + lease_time)
    id_str = utils.ProcessIdString()

    query = ("UPDATE cron_jobs "
             "SET leased_until=FROM_UNIXTIME(%s), leased_by=%s "
             "WHERE (leased_until IS NULL OR leased_until < FROM_UNIXTIME(%s))")
    args = [expiry_str, id_str, now_str]

    if cronjob_ids:
      query += " AND job_id in (%s)" % ", ".join(["%s"] * len(cronjob_ids))
      args += cronjob_ids

    updated = cursor.execute(query, args)

    if updated == 0:
      return []

    cursor.execute(
        "SELECT job, UNIX_TIMESTAMP(create_time), enabled,"
        "forced_run_requested, last_run_status, UNIX_TIMESTAMP(last_run_time), "
        "current_run_id, state, UNIX_TIMESTAMP(leased_until), leased_by "
        "FROM cron_jobs "
        "FORCE INDEX (cron_jobs_by_lease) "
        "WHERE leased_until=FROM_UNIXTIME(%s) AND leased_by=%s",
        [expiry_str, id_str])
    return [self._CronJobFromRow(row) for row in cursor.fetchall()]

Response: Leases all available cron jobs.


Instruction:
Make a summary of the following Python 3 code
def ReturnLeasedCronJobs(self, jobs, cursor=None):
    if not jobs:
      return

    unleased_jobs = []

    conditions = []
    args = []
    for job in jobs:
      if not job.leased_by or not job.leased_until:
        unleased_jobs.append(job)
        continue

      conditions.append("""
      (job_id = %s AND leased_until = FROM_UNIXTIME(%s) AND leased_by = %s)
      """)
      args += [
          job.cron_job_id,
          mysql_utils.RDFDatetimeToTimestamp(job.leased_until), job.leased_by
      ]

    if conditions:
      query = ("UPDATE cron_jobs "
               "SET leased_until=NULL, leased_by=NULL "
               "WHERE ") + " OR ".join(conditions)
      returned = cursor.execute(query, args)

    if unleased_jobs:
      raise ValueError("CronJobs to return are not leased: %s" % unleased_jobs)
    if returned != len(jobs):
      raise ValueError("%d cronjobs in %s could not be returned." %
                       ((len(jobs) - returned), jobs))

Response: Returns the leased cron jobs.


Instruction:
Can you tell what is the following Python 3 function doing
def WriteCronJobRun(self, run_object, cursor=None):
    query = ("INSERT INTO cron_job_runs "
             "(job_id, run_id, write_time, run) "
             "VALUES (%s, %s, FROM_UNIXTIME(%s), %s) "
             "ON DUPLICATE KEY UPDATE "
             "run=VALUES(run), write_time=VALUES(write_time)")

    write_time_str = mysql_utils.RDFDatetimeToTimestamp(
        rdfvalue.RDFDatetime.Now())
    try:
      cursor.execute(query, [
          run_object.cron_job_id,
          db_utils.CronJobRunIDToInt(run_object.run_id),
          write_time_str,
          run_object.SerializeToString(),
      ])
    except MySQLdb.IntegrityError as e:
      raise db.UnknownCronJobError(
          "CronJob with id %s not found." % run_object.cron_job_id, cause=e)

Response: Writes a cron job run object to the database.


Instruction:
Create a Python 3 function for
reading all cron job runs for a given job id.

Response:
def ReadCronJobRuns(self, job_id, cursor=None):
    """Reads all cron job runs for a given job id."""
    query = """
    SELECT run, UNIX_TIMESTAMP(write_time)
      FROM cron_job_runs
     WHERE job_id = %s
    """
    cursor.execute(query, [job_id])
    runs = [self._CronJobRunFromRow(row) for row in cursor.fetchall()]
    return sorted(runs, key=lambda run: run.started_at, reverse=True)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def ReadCronJobRun(self, job_id, run_id, cursor=None):
    query = ("SELECT run, UNIX_TIMESTAMP(write_time) FROM cron_job_runs "
             "WHERE job_id = %s AND run_id = %s")
    num_runs = cursor.execute(
        query, [job_id, db_utils.CronJobRunIDToInt(run_id)])
    if num_runs == 0:
      raise db.UnknownCronJobRunError(
          "Run with job id %s and run id %s not found." % (job_id, run_id))

    return self._CronJobRunFromRow(cursor.fetchall()[0])

Response: Reads a single cron job run from the db.


Instruction:
Can you write a function in Python 3 where it
deletes cron job runs that are older than the given timestamp.

Response:
def DeleteOldCronJobRuns(self, cutoff_timestamp, cursor=None):
    """Deletes cron job runs that are older then the given timestamp."""
    query = "DELETE FROM cron_job_runs WHERE write_time < FROM_UNIXTIME(%s)"
    cursor.execute(query,
                   [mysql_utils.RDFDatetimeToTimestamp(cutoff_timestamp)])


Instruction:
Can you tell what is the following Python 3 function doing
def _Insert(cursor, table, values):
  precondition.AssertIterableType(values, dict)

  if not values:  # Nothing can be INSERTed with empty `values` list.
    return

  column_names = list(sorted(values[0]))
  for value_dict in values:
    if set(column_names) != set(value_dict):
      raise ValueError("Given value dictionaries must have identical keys. "
                       "Expecting columns {!r}, but got value {!r}".format(
                           column_names, value_dict))

  query = "INSERT IGNORE INTO %s {cols} VALUES {vals}" % table
  query = query.format(
      cols=mysql_utils.Columns(column_names),
      vals=mysql_utils.Placeholders(num=len(column_names), values=len(values)))

  values_list = []
  for values_dict in values:
    values_list.extend(values_dict[column] for column in column_names)

  cursor.execute(query, values_list)

Response: Insert one or multiple rows into the given table.


Instruction:
Given the following Python 3 function, write the documentation
def _BlobToChunks(blob_id, blob):
  #  In case of empty blob (with empty range), use [0].
  chunk_begins = list(range(0, len(blob), BLOB_CHUNK_SIZE)) or [0]
  chunks = []
  for i, chunk_begin in enumerate(chunk_begins):
    chunks.append({
        "blob_id": blob_id,
        "chunk_index": i,
        "blob_chunk": blob[chunk_begin:chunk_begin + BLOB_CHUNK_SIZE]
    })
  return chunks

Response: Splits a Blob into chunks of size BLOB_CHUNK_SIZE.


Instruction:
Implement a function in Python 3 to
group chunks into partitions of size safe for a single INSERT.

Response:
def _PartitionChunks(chunks):
  """Groups chunks into partitions of size safe for a single INSERT."""
  partitions = [[]]
  partition_size = 0

  for chunk in chunks:
    cursize = len(chunk["blob_chunk"])
    if (cursize + partition_size > BLOB_CHUNK_SIZE or
        len(partitions[-1]) >= CHUNKS_PER_INSERT):
      partitions.append([])
      partition_size = 0
    partitions[-1].append(chunk)
    partition_size += cursize

  return partitions


Instruction:
How would you implement a function in Python 3 that
checks if given blobs exist.

Response:
def CheckBlobsExist(self, blob_ids, cursor=None):
    """Checks if given blobs exist."""
    if not blob_ids:
      return {}

    exists = {blob_id: False for blob_id in blob_ids}
    query = ("SELECT blob_id "
             "FROM blobs "
             "FORCE INDEX (PRIMARY) "
             "WHERE blob_id IN {}".format(
                 mysql_utils.Placeholders(len(blob_ids))))
    cursor.execute(query, [blob_id.AsBytes() for blob_id in blob_ids])
    for blob_id, in cursor.fetchall():
      exists[rdf_objects.BlobID.FromBytes(blob_id)] = True
    return exists


Instruction:
How would you implement a function in Python 3 that
writes blob references for a given set of hashes.

Response:
def WriteHashBlobReferences(self, references_by_hash, cursor):
    """Writes blob references for a given set of hashes."""
    values = []
    for hash_id, blob_refs in iteritems(references_by_hash):
      refs = rdf_objects.BlobReferences(items=blob_refs).SerializeToString()
      values.append({
          "hash_id": hash_id.AsBytes(),
          "blob_references": refs,
      })
    _Insert(cursor, "hash_blob_references", values)


Instruction:
Can you generate the documentation for the following Python 3 function
def ReadHashBlobReferences(self, hashes, cursor):
    query = ("SELECT hash_id, blob_references FROM hash_blob_references WHERE "
             "hash_id IN {}").format(mysql_utils.Placeholders(len(hashes)))
    cursor.execute(query, [hash_id.AsBytes() for hash_id in hashes])
    results = {hash_id: None for hash_id in hashes}
    for hash_id, blob_references in cursor.fetchall():
      sha_hash_id = rdf_objects.SHA256HashID.FromBytes(hash_id)
      refs = rdf_objects.BlobReferences.FromSerializedString(blob_references)
      results[sha_hash_id] = list(refs.items)
    return results

Response: Reads blob references of a given set of hashes.


Instruction:
How would you explain what the following Python 3 function does
def WriteBlobsWithUnknownHashes(
      self, blobs_data):
    """Calculates hash ids and writes contents of given data blobs.

    Args:
      blobs_data: An iterable of bytes.

    Returns:
      A list of rdf_objects.BlobID objects with each blob id corresponding
      to an element in the original blobs_data argument.
    """
    blobs_ids = [rdf_objects.BlobID.FromBlobData(d) for d in blobs_data]
    self.WriteBlobs(dict(zip(blobs_ids, blobs_data)))
    return blobs_ids

Response: Calculates hash ids and writes contents of given blobs.


Instruction:
Implement a Python 3 function for
returning the client id from the queue q.

Response:
def _GetClientIdFromQueue(q):
  """Returns q's client id, if q is a client task queue, otherwise None.

  Args:
    q: rdfvalue.RDFURN

  Returns:
    string or None
  """
  split = q.Split()
  if not split or len(split) < 2:
    return None

  # Normalize to lowercase.
  split = [s.lower() for s in split]

  str_client_id, tasks_marker = split

  if not str_client_id.startswith("c.") or tasks_marker != "tasks":
    return None

  # The "C." prefix should be uppercase.
  str_client_id = "C" + str_client_id[1:]

  return str_client_id


Instruction:
Write a Python 3 function for
getting a single shard for a given queue.

Response:
def GetNotificationShard(self, queue):
    """Gets a single shard for a given queue."""
    queue_name = str(queue)
    QueueManager.notification_shard_counters.setdefault(queue_name, 0)
    QueueManager.notification_shard_counters[queue_name] += 1
    notification_shard_index = (
        QueueManager.notification_shard_counters[queue_name] %
        self.num_notification_shards)
    if notification_shard_index > 0:
      return queue.Add(str(notification_shard_index))
    else:
      return queue


Instruction:
Implement a Python 3 function for
returning a copy of the QueueManager object.

Response:
def Copy(self):
    """Return a copy of the queue manager.

    Returns:
    Copy of the QueueManager object.
    NOTE: pending writes/deletions are not copied. On the other hand, if the
    original object has a frozen timestamp, a copy will have it as well.
    """
    result = QueueManager(store=self.data_store, token=self.token)
    result.prev_frozen_timestamps = self.prev_frozen_timestamps
    result.frozen_timestamp = self.frozen_timestamp
    return result


Instruction:
Can you generate a brief explanation for the following Python 3 code
def FreezeTimestamp(self):
    self.prev_frozen_timestamps.append(self.frozen_timestamp)
    self.frozen_timestamp = rdfvalue.RDFDatetime.Now()

Response: Freeze the timestamp used for resolve and delete database queries.


Instruction:
Create a Python 3 function for
fetching all the requests with a status message queued for them.

Response:
def FetchCompletedRequests(self, session_id, timestamp=None):
    """Fetch all the requests with a status message queued for them."""
    if timestamp is None:
      timestamp = (0, self.frozen_timestamp or rdfvalue.RDFDatetime.Now())

    for request, status in self.data_store.ReadCompletedRequests(
        session_id, timestamp=timestamp, limit=self.request_limit):
      yield request, status


Instruction:
Can you tell what is the following Python 3 function doing
def FetchCompletedResponses(self, session_id, timestamp=None, limit=10000):
    if timestamp is None:
      timestamp = (0, self.frozen_timestamp or rdfvalue.RDFDatetime.Now())

    completed_requests = collections.deque(
        self.FetchCompletedRequests(session_id, timestamp=timestamp))

    total_size = 0
    while completed_requests:

      # Size reported in the status messages may be different from actual
      # number of responses read from the database. Example: hunt responses
      # may get deleted from the database and then worker may die before
      # deleting the request. Then status.response_id will be >0, but no
      # responses will be read from the DB.
      projected_total_size = total_size
      request_list = []
      while completed_requests:
        request, status = completed_requests.popleft()

        # Make sure at least one response is fetched.
        request_list.append(request)

        # Quit if there are too many responses.
        projected_total_size += status.response_id
        if projected_total_size > limit:
          break

      for request, responses in self.data_store.ReadResponses(request_list):

        yield (request, responses)
        total_size += len(responses)
        if total_size > limit:
          raise MoreDataException()

Response: Fetch only completed requests and responses up to a limit.


Instruction:
Create a Python 3 function to
fetch all outstanding requests and responses for this flow.

Response:
def FetchRequestsAndResponses(self, session_id, timestamp=None):
    """Fetches all outstanding requests and responses for this flow.

    We first cache all requests and responses for this flow in memory to
    prevent round trips.

    Args:
      session_id: The session_id to get the requests/responses for.
      timestamp: Tuple (start, end) with a time range. Fetched requests and
        responses will have timestamp in this range.

    Yields:
      an tuple (request protobufs, list of responses messages) in ascending
      order of request ids.

    Raises:
      MoreDataException: When there is more data available than read by the
                         limited query.
    """
    if timestamp is None:
      timestamp = (0, self.frozen_timestamp or rdfvalue.RDFDatetime.Now())

    num_requests = 0
    for request, responses in self.data_store.ReadRequestsAndResponses(
        session_id,
        timestamp=timestamp,
        request_limit=self.request_limit,
        response_limit=self.response_limit):
      yield (request, responses)
      num_requests += 1

    if num_requests >= self.request_limit:
      raise MoreDataException()


Instruction:
Write a Python 3 function that can
delete the request and all its responses from the flow state queue.

Response:
def DeleteRequest(self, request):
    """Deletes the request and all its responses from the flow state queue."""
    self.requests_to_delete.append(request)

    if request and request.HasField("request"):
      self.DeQueueClientRequest(request.request)

    data_store.DB.DeleteRequest(request)


Instruction:
Can you tell what is the following Python 3 function doing
def MultiDestroyFlowStates(self, session_ids):
    deleted_requests = self.data_store.MultiDestroyFlowStates(
        session_ids, request_limit=self.request_limit)

    for request in deleted_requests:
      if request.HasField("request"):
        # Client request dequeueing is cached so we can call it directly.
        self.DeQueueClientRequest(request.request)

Response: Deletes all states in multiple flows and dequeues all client messages.


Instruction:
Explain what the following Python 3 code does
def Flush(self):

    self.data_store.StoreRequestsAndResponses(
        new_requests=self.request_queue,
        new_responses=self.response_queue,
        requests_to_delete=self.requests_to_delete)

    # We need to make sure that notifications are written after the requests so
    # we flush after writing all requests and only notify afterwards.
    mutation_pool = self.data_store.GetMutationPool()
    with mutation_pool:

      messages_by_queue = collection.Group(
          list(itervalues(
              self.client_messages_to_delete)), lambda request: request.queue)
      for queue, messages in iteritems(messages_by_queue):
        self.Delete(queue, messages, mutation_pool=mutation_pool)

      if self.new_client_messages:
        for timestamp, messages in iteritems(
            collection.Group(self.new_client_messages, lambda x: x[1])):

          self.Schedule([x[0] for x in messages],
                        timestamp=timestamp,
                        mutation_pool=mutation_pool)

    if self.notifications:
      for notification in itervalues(self.notifications):
        self.NotifyQueue(notification, mutation_pool=mutation_pool)

      mutation_pool.Flush()

    self.request_queue = []
    self.response_queue = []
    self.requests_to_delete = []

    self.client_messages_to_delete = {}
    self.notifications = {}
    self.new_client_messages = []

Response: Flushes all the changes in this object to the datastore.


Instruction:
Write a Python 3 function that can
queue the message on the flow s state.

Response:
def QueueResponse(self, response, timestamp=None):
    """Queues the message on the flow's state."""
    if timestamp is None:
      timestamp = self.frozen_timestamp
    self.response_queue.append((response, timestamp))


Instruction:
Can you create a Python 3 function that
queues a notification for a flow.

Response:
def QueueNotification(self, notification=None, timestamp=None, **kw):
    """Queues a notification for a flow."""

    if notification is None:
      notification = rdf_flows.GrrNotification(**kw)

    session_id = notification.session_id

    if session_id:
      if timestamp is None:
        timestamp = self.frozen_timestamp

      notification.timestamp = timestamp

      # We must not store more than one notification per session id and
      # timestamp or there will be race conditions. We therefore only keep
      # the one with the highest request number (indicated by last_status).
      # Note that timestamp might be None. In that case we also only want
      # to keep the latest.
      if timestamp is None:
        ts_str = "None"
      else:
        ts_str = int(timestamp)
      key = "%s!%s" % (session_id, ts_str)
      existing = self.notifications.get(key)
      if existing is not None:
        if existing.last_status < notification.last_status:
          self.notifications[key] = notification
      else:
        self.notifications[key] = notification


Instruction:
Here you have a function in Python 3, explain what it does
def Delete(self, queue, tasks, mutation_pool=None):
    if queue is None:
      return
    if mutation_pool is None:
      raise ValueError("Mutation pool can't be none.")
    mutation_pool.QueueDeleteTasks(queue, tasks)

Response: Deletes the tasks from the queue.


Instruction:
Can you create a Python 3 function that
schedules a set of Task instances.

Response:
def Schedule(self, tasks, mutation_pool, timestamp=None):
    """Schedule a set of Task() instances."""
    non_fleetspeak_tasks = []
    for queue, queued_tasks in iteritems(
        collection.Group(tasks, lambda x: x.queue)):
      if not queue:
        continue

      client_id = _GetClientIdFromQueue(queue)
      if fleetspeak_utils.IsFleetspeakEnabledClient(
          client_id, token=self.token):
        for task in queued_tasks:
          fleetspeak_utils.SendGrrMessageThroughFleetspeak(client_id, task)
        continue
      non_fleetspeak_tasks.extend(queued_tasks)

    timestamp = timestamp or self.frozen_timestamp
    mutation_pool.QueueScheduleTasks(non_fleetspeak_tasks, timestamp)


Instruction:
Implement a function in Python 3 to
return all queue notifications.

Response:
def GetNotifications(self, queue):
    """Returns all queue notifications."""
    queue_shard = self.GetNotificationShard(queue)
    return self._GetUnsortedNotifications(queue_shard).values()


Instruction:
How would you explain what the following Python 3 function does
def GetNotificationsForAllShards(self, queue):
    notifications_by_session_id = {}
    for queue_shard in self.GetAllNotificationShards(queue):
      self._GetUnsortedNotifications(
          queue_shard, notifications_by_session_id=notifications_by_session_id)

    return notifications_by_session_id.values()

Response: Returns all notifications for all shards of a queue at once.


Instruction:
Can you implement a function in Python 3 that
returns all the unsorted notifications for a queue_shard.

Response:
def _GetUnsortedNotifications(self,
                                queue_shard,
                                notifications_by_session_id=None):
    """Returns all the available notifications for a queue_shard.

    Args:
      queue_shard: urn of queue shard
      notifications_by_session_id: store notifications in this dict rather than
        creating a new one

    Returns:
      dict of notifications. keys are session ids.
    """
    if notifications_by_session_id is None:
      notifications_by_session_id = {}
    end_time = self.frozen_timestamp or rdfvalue.RDFDatetime.Now()
    for notification in self.data_store.GetNotifications(queue_shard, end_time):

      existing = notifications_by_session_id.get(notification.session_id)
      if existing:
        # If we have a notification for this session_id already, we only store
        # the one that was scheduled last.
        if notification.first_queued > existing.first_queued:
          notifications_by_session_id[notification.session_id] = notification
        elif notification.first_queued == existing.first_queued and (
            notification.last_status > existing.last_status):
          # Multiple notifications with the same timestamp should not happen.
          # We can still do the correct thing and use the latest one.
          logging.warning(
              "Notifications with equal first_queued fields detected: %s %s",
              notification, existing)
          notifications_by_session_id[notification.session_id] = notification
      else:
        notifications_by_session_id[notification.session_id] = notification

    return notifications_by_session_id


Instruction:
Given the following Python 3 function, write the documentation
def MultiNotifyQueue(self, notifications, mutation_pool=None):
    extract_queue = lambda notification: notification.session_id.Queue()
    for queue, notifications in iteritems(
        collection.Group(notifications, extract_queue)):
      self._MultiNotifyQueue(queue, notifications, mutation_pool=mutation_pool)

Response: This is the same as NotifyQueue but for several session_ids at once.


Instruction:
Can you tell what is the following Python 3 function doing
def _MultiNotifyQueue(self, queue, notifications, mutation_pool=None):
    notification_list = []
    now = rdfvalue.RDFDatetime.Now()
    for notification in notifications:
      if not notification.first_queued:
        notification.first_queued = (
            self.frozen_timestamp or rdfvalue.RDFDatetime.Now())
      else:
        diff = now - notification.first_queued
        if diff.seconds >= self.notification_expiry_time:
          # This notification has been around for too long, we drop it.
          logging.debug("Dropping notification: %s", str(notification))
          continue

      notification_list.append(notification)

    mutation_pool.CreateNotifications(
        self.GetNotificationShard(queue), notification_list)

Response: Does the actual queuing.


Instruction:
How would you implement a function in Python 3 that
retrieves all the tasks from a queue without leasing them.

Response:
def Query(self, queue, limit=1):
    """Retrieves tasks from a queue without leasing them.

    This is good for a read only snapshot of the tasks.

    Args:
       queue: The task queue that this task belongs to, usually client.Queue()
         where client is the ClientURN object you want to schedule msgs on.
       limit: Number of values to fetch.

    Returns:
        A list of Task() objects.
    """
    # This function is usually used for manual testing so we also accept client
    # ids and get the queue from it.
    if isinstance(queue, rdf_client.ClientURN):
      queue = queue.Queue()

    return self.data_store.QueueQueryTasks(queue, limit=limit)


Instruction:
How would you explain what the following Python 3 function does
def QueryAndOwn(self, queue, lease_seconds=10, limit=1):
    with self.data_store.GetMutationPool() as mutation_pool:
      return mutation_pool.QueueQueryAndOwn(queue, lease_seconds, limit,
                                            self.frozen_timestamp)

Response: Returns a list of GrrMessage objects leased for a certain time.


Instruction:
How would you explain what the following Python 3 function does
def FetchResponses(self, session_id):
    timestamp = (0, self.frozen_timestamp or rdfvalue.RDFDatetime.Now())

    for response in self.data_store.FetchResponsesForWellKnownFlow(
        session_id, self.response_limit, timestamp=timestamp):
      yield response

Response: Retrieves the responses for a well known flow.


Instruction:
Write a Python 3 script for
migrating one Artifact from AFF4 to REL_DB.

Response:
def _MigrateArtifact(artifact):
  """Migrate one Artifact from AFF4 to REL_DB."""
  name = Text(artifact.name)

  try:
    logging.info("Migating %s", name)
    data_store.REL_DB.WriteArtifact(artifact)
    logging.info("  Wrote %s", name)
  except db.DuplicatedArtifactError:
    logging.info("  Skipped %s, because artifact already exists.", name)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def MigrateArtifacts():

  # First, delete all existing artifacts in REL_DB.
  artifacts = data_store.REL_DB.ReadAllArtifacts()
  if artifacts:
    logging.info("Deleting %d artifacts from REL_DB.", len(artifacts))
    for artifact in data_store.REL_DB.ReadAllArtifacts():
      data_store.REL_DB.DeleteArtifact(Text(artifact.name))
  else:
    logging.info("No artifacts found in REL_DB.")

  artifacts = artifact_registry.REGISTRY.GetArtifacts(
      reload_datastore_artifacts=True)

  logging.info("Found %d artifacts in AFF4.", len(artifacts))

  # Only migrate user-created artifacts.
  artifacts = list(filter(_IsCustom, artifacts))

  logging.info("Migrating %d user-created artifacts.", len(artifacts))

  for artifact in artifacts:
    _MigrateArtifact(artifact)

Response: Migrates Artifacts from AFF4 to REL_DB.


Instruction:
Here you have a function in Python 3, explain what it does
def _GetBaseURLs(self):
    result = config.CONFIG["Client.server_urls"]
    if not result:
      # Backwards compatibility - deduce server_urls from Client.control_urls.
      for control_url in config.CONFIG["Client.control_urls"]:
        result.append(posixpath.dirname(control_url) + "/")

    # Check the URLs for trailing /. This traps configuration errors.
    for url in result:
      if not url.endswith("/"):
        raise RuntimeError("Bad URL: %s URLs must end with /" % url)

    return result

Response: Gathers a list of base URLs we will try.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _GetProxies(self):
    # Detect proxies from the OS environment.
    result = client_utils.FindProxies()

    # Also try to connect directly if all proxies fail.
    result.append("")

    # Also try all proxies configured in the config system.
    result.extend(config.CONFIG["Client.proxy_servers"])

    return result

Response: Gather a list of proxies to use.


Instruction:
Write a Python 3 function for
opening a server endpoint.

Response:
def OpenServerEndpoint(self,
                         path,
                         verify_cb=lambda x: True,
                         data=None,
                         params=None,
                         headers=None,
                         method="GET",
                         timeout=None):
    """Search through all the base URLs to connect to one that works.

    This is a thin wrapper around requests.request() so most parameters are
    documented there.

    Args:
      path: The URL path to access in this endpoint.
      verify_cb: A callback which should return True if the response is
        reasonable. This is used to detect if we are able to talk to the correct
        endpoint. If not we try a different endpoint/proxy combination.
      data: Parameters to send in POST bodies (See Requests documentation).
      params: Parameters to send in GET URLs (See Requests documentation).
      headers: Additional headers (See Requests documentation)
      method: The HTTP method to use. If not set we select one automatically.
      timeout: See Requests documentation.

    Returns:
      an HTTPObject() instance with the correct error code set.
    """
    tries = 0
    last_error = HTTPObject(code=404)

    while tries < len(self.base_urls):
      base_url_index = self.last_base_url_index % len(self.base_urls)
      active_base_url = self.base_urls[base_url_index]

      result = self.OpenURL(
          self._ConcatenateURL(active_base_url, path),
          data=data,
          params=params,
          headers=headers,
          method=method,
          timeout=timeout,
          verify_cb=verify_cb,
      )

      if not result.Success():
        tries += 1
        self.last_base_url_index += 1
        last_error = result
        continue

      # The URL worked - we record that.
      self.active_base_url = active_base_url

      return result

    # No connection is possible at all.
    logging.info(
        "Could not connect to GRR servers %s, directly or through "
        "these proxies: %s.", self.base_urls, self.proxies)

    return last_error