Instruction:
Explain what the following Python 3 code does
def _GetSqliteSchema(self, proto_struct_class, prefix=""):
    schema = collections.OrderedDict()
    for type_info in proto_struct_class.type_infos:
      if type_info.__class__ is rdf_structs.ProtoEmbedded:
        schema.update(
            self._GetSqliteSchema(
                type_info.type, prefix="%s%s." % (prefix, type_info.name)))
      else:
        field_name = utils.SmartStr(prefix + type_info.name)
        schema[field_name] = Rdf2SqliteAdapter.GetConverter(type_info)
    return schema

Response: Returns a mapping of SQLite column names to Converter objects.


Instruction:
Can you generate the documentation for the following Python 3 function
def _ConvertToCanonicalSqlDict(self, schema, raw_dict, prefix=""):
    flattened_dict = {}
    for k, v in iteritems(raw_dict):
      if isinstance(v, dict):
        flattened_dict.update(
            self._ConvertToCanonicalSqlDict(
                schema, v, prefix="%s%s." % (prefix, k)))
      else:
        field_name = prefix + k
        flattened_dict[field_name] = schema[field_name].convert_fn(v)
    return flattened_dict

Response: Converts a dict of RDF values into a SQL - ready form.


Instruction:
Can you generate the documentation for the following Python 3 function
def _FlushAllRows(self, db_connection, table_name):
    for sql in db_connection.iterdump():
      if (sql.startswith("CREATE TABLE") or
          sql.startswith("BEGIN TRANSACTION") or sql.startswith("COMMIT")):
        # These statements only need to be written once.
        continue
      # The archive generator expects strings (not Unicode objects returned by
      # the pysqlite library).
      yield self.archive_generator.WriteFileChunk((sql + "\n").encode("utf-8"))
    with db_connection:
      db_connection.cursor().execute("DELETE FROM \"%s\";" % table_name)

Response: Copies all rows from the given db into the output file then deletes them.


Instruction:
Implement a function in Python 3 to
dump the given aff4object into a yaml representation.

Response:
def YamlDumper(aff4object):
  """Dumps the given aff4object into a yaml representation."""
  aff4object.Flush()

  result = {}
  for attribute, values in iteritems(aff4object.synced_attributes):
    result[attribute.predicate] = []
    for value in values:
      # This value is really a LazyDecoder() instance. We need to get at the
      # real data here.
      value = value.ToRDFValue()

      result[attribute.predicate].append(
          [value.__class__.__name__,
           value.SerializeToString(),
           str(value.age)])

  return yaml.Dump({
      "aff4_class": compatibility.GetName(aff4object),
      "_urn": aff4object.urn.SerializeToString(),
      "attributes": result,
      "age_policy": aff4object.age_policy,
  })


Instruction:
Can you tell what is the following Python 3 function doing
def YamlLoader(string):
  representation = yaml.Parse(string)
  result_cls = aff4.FACTORY.AFF4Object(representation["aff4_class"])
  aff4_attributes = {}
  for predicate, values in iteritems(representation["attributes"]):
    attribute = aff4.Attribute.PREDICATES[predicate]
    tmp = aff4_attributes[attribute] = []

    for rdfvalue_cls_name, value, age in values:
      rdfvalue_cls = aff4.FACTORY.RDFValue(rdfvalue_cls_name)
      value = rdfvalue_cls(value, age=rdfvalue.RDFDatetime(age))
      tmp.append(value)

  # Ensure the object is dirty so when we save it, it can be written to the data
  # store.
  result = result_cls(
      urn=representation["_urn"],
      clone=aff4_attributes,
      mode="rw",
      age=representation["age_policy"])

  result.new_attributes, result.synced_attributes = result.synced_attributes, {}

  result._dirty = True  # pylint: disable=protected-access

  return result

Response: Load an AFF4 object from a serialized YAML representation.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def GetMetadata(client_id, client_full_info):

  metadata = ExportedMetadata()

  last_snapshot = None
  if client_full_info.HasField("last_snapshot"):
    last_snapshot = client_full_info.last_snapshot

  metadata.client_urn = client_id
  metadata.client_age = client_full_info.metadata.first_seen

  if last_snapshot is not None:
    kb = client_full_info.last_snapshot.knowledge_base

    metadata.hostname = kb.fqdn
    metadata.os = kb.os
    metadata.uname = last_snapshot.Uname()
    metadata.os_release = last_snapshot.os_release
    metadata.os_version = last_snapshot.os_version
    metadata.usernames = ",".join(user.username for user in kb.users)

    addresses = last_snapshot.GetMacAddresses()
    if addresses:
      metadata.mac_address = last_snapshot.GetMacAddresses().pop()
    metadata.hardware_info = last_snapshot.hardware_info
    metadata.kernel_version = last_snapshot.kernel

    ci = last_snapshot.cloud_instance
    if ci is not None:
      if ci.cloud_type == ci.InstanceType.AMAZON:
        metadata.cloud_instance_type = metadata.CloudInstanceType.AMAZON
        metadata.cloud_instance_id = ci.amazon.instance_id
      elif ci.cloud_type == ci.InstanceType.GOOGLE:
        metadata.cloud_instance_type = metadata.CloudInstanceType.GOOGLE
        metadata.cloud_instance_id = ci.google.unique_id

  system_labels = set()
  user_labels = set()
  for l in client_full_info.labels:
    if l.owner == "GRR":
      system_labels.add(l.name)
    else:
      user_labels.add(l.name)

  metadata.labels = ",".join(sorted(system_labels | user_labels))
  metadata.system_labels = ",".join(sorted(system_labels))
  metadata.user_labels = ",".join(sorted(user_labels))

  return metadata

Response: Builds ExportedMetadata object for a given client id and ClientFullInfo object.


Instruction:
Can you tell what is the following Python 3 function doing
def GetMetadataLegacy(client, token=None):
  if isinstance(client, rdfvalue.RDFURN):
    client_fd = aff4.FACTORY.Open(client, mode="r", token=token)
  else:
    client_fd = client

  metadata = ExportedMetadata()

  metadata.client_urn = client_fd.urn
  metadata.client_age = client_fd.urn.age

  metadata.hostname = utils.SmartUnicode(
      client_fd.Get(client_fd.Schema.HOSTNAME, ""))

  metadata.os = utils.SmartUnicode(client_fd.Get(client_fd.Schema.SYSTEM, ""))

  metadata.uname = utils.SmartUnicode(client_fd.Get(client_fd.Schema.UNAME, ""))

  metadata.os_release = utils.SmartUnicode(
      client_fd.Get(client_fd.Schema.OS_RELEASE, ""))

  metadata.os_version = utils.SmartUnicode(
      client_fd.Get(client_fd.Schema.OS_VERSION, ""))

  kb = client_fd.Get(client_fd.Schema.KNOWLEDGE_BASE)
  usernames = ""
  if kb:
    usernames = [user.username for user in kb.users] or ""
  metadata.usernames = utils.SmartUnicode(usernames)

  metadata.mac_address = utils.SmartUnicode(
      client_fd.Get(client_fd.Schema.MAC_ADDRESS, ""))

  system_labels = set()
  user_labels = set()
  for l in client_fd.GetLabels():
    if l.owner == "GRR":
      system_labels.add(l.name)
    else:
      user_labels.add(l.name)

  metadata.labels = ",".join(sorted(system_labels | user_labels))

  metadata.system_labels = ",".join(sorted(system_labels))

  metadata.user_labels = ",".join(sorted(user_labels))

  metadata.hardware_info = client_fd.Get(client_fd.Schema.HARDWARE_INFO)

  metadata.kernel_version = client_fd.Get(client_fd.Schema.KERNEL)

  return metadata

Response: Builds an ExportedMetadata object for a given client id.


Instruction:
How would you code a function in Python 3 to
convert a set of RDFValues into a set of export - friendly RDFValues.

Response:
def ConvertValuesWithMetadata(metadata_value_pairs, token=None, options=None):
  """Converts a set of RDFValues into a set of export-friendly RDFValues.

  Args:
    metadata_value_pairs: Tuples of (metadata, rdf_value), where metadata is an
      instance of ExportedMetadata and rdf_value is an RDFValue subclass
      instance to be exported.
    token: Security token.
    options: rdfvalue.ExportOptions instance that will be passed to
      ExportConverters.

  Yields:
    Converted values. Converted values may be of different types.

  Raises:
    NoConverterFound: in case no suitable converters were found for a value in
                      metadata_value_pairs. This error is only raised after
                      all values in metadata_value_pairs are attempted to be
                      converted. If there are multiple value types that could
                      not be converted because of the lack of corresponding
                      converters, only the last one will be specified in the
                      exception message.
  """
  no_converter_found_error = None
  for metadata_values_group in itervalues(
      collection.Group(
          metadata_value_pairs, lambda pair: pair[1].__class__.__name__)):

    _, first_value = metadata_values_group[0]
    converters_classes = ExportConverter.GetConvertersByValue(first_value)
    if not converters_classes:
      no_converter_found_error = "No converters found for value: %s" % str(
          first_value)
      continue

    converters = [cls(options) for cls in converters_classes]
    for converter in converters:
      for result in converter.BatchConvert(metadata_values_group, token=token):
        yield result

  if no_converter_found_error is not None:
    raise NoConverterFound(no_converter_found_error)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def ConvertValues(default_metadata, values, token=None, options=None):
  batch_data = [(default_metadata, obj) for obj in values]
  return ConvertValuesWithMetadata(batch_data, token=token, options=options)

Response: Converts a set of RDFValues into a set of export - friendly RDFValues.


Instruction:
Can you tell what is the following Python 3 function doing
def GetConvertersByClass(value_cls):
    try:
      return ExportConverter.converters_cache[value_cls]
    except KeyError:
      results = [
          cls for cls in itervalues(ExportConverter.classes)
          if cls.input_rdf_type == value_cls
      ]
      if not results:
        results = [DataAgnosticExportConverter]

      ExportConverter.converters_cache[value_cls] = results
      return results

Response: Returns all converters that take given value as an input value.


Instruction:
Create a Python 3 function for
generating a flattened RDFValue class definition for the given value.

Response:
def MakeFlatRDFClass(self, value):
    """Generates flattened RDFValue class definition for the given value."""

    def Flatten(self, metadata, value_to_flatten):
      if metadata:
        self.metadata = metadata

      for desc in value_to_flatten.type_infos:
        if desc.name == "metadata":
          continue
        if hasattr(self, desc.name) and value_to_flatten.HasField(desc.name):
          setattr(self, desc.name, getattr(value_to_flatten, desc.name))

    descriptors = []
    enums = {}

    # Metadata is always the first field of exported data.
    descriptors.append(
        rdf_structs.ProtoEmbedded(
            name="metadata", field_number=1, nested=ExportedMetadata))

    for number, desc in sorted(iteritems(value.type_infos_by_field_number)):
      # Name 'metadata' is reserved to store ExportedMetadata value.
      if desc.name == "metadata":
        logging.debug("Ignoring 'metadata' field in %s.",
                      value.__class__.__name__)
        continue

      # Copy descriptors for primivie values as-is, just make sure their
      # field number is correct.
      if isinstance(desc, (rdf_structs.ProtoBinary, rdf_structs.ProtoString,
                           rdf_structs.ProtoUnsignedInteger,
                           rdf_structs.ProtoRDFValue, rdf_structs.ProtoEnum)):
        # Incrementing field number by 1, as 1 is always occuppied by metadata.
        descriptors.append(desc.Copy(field_number=number + 1))

      if (isinstance(desc, rdf_structs.ProtoEnum) and
          not isinstance(desc, rdf_structs.ProtoBoolean)):
        # Attach the enum container to the class for easy reference:
        enums[desc.enum_name] = desc.enum_container

    # Create the class as late as possible. This will modify a
    # metaclass registry, we need to make sure there are no problems.
    output_class = type(
        self.ExportedClassNameForValue(value), (AutoExportedProtoStruct,),
        dict(Flatten=Flatten))

    for descriptor in descriptors:
      output_class.AddDescriptor(descriptor)

    for name, container in iteritems(enums):
      setattr(output_class, name, container)

    return output_class


Instruction:
How would you explain what the following Python 3 function does
def ParseSignedData(signed_data, result):
    try:
      auth_data
    except NameError:
      # Verify_sigs is not available so we can't parse signatures. If you want
      # this functionality, please install the verify-sigs package:
      # https://github.com/anthrotype/verify-sigs
      # TODO(amoser): Make verify-sigs a pip package and add a dependency.
      return

    try:
      try:
        auth = auth_data.AuthData(signed_data.certificate)
      except Exception as e:  # pylint: disable=broad-except
        # If we failed to parse the certificate, we want the user to know it.
        result.cert_hasher_name = "Error parsing certificate: %s" % str(e)
        raise

      result.cert_hasher_name = auth.digest_algorithm().name
      result.cert_program_name = str(auth.program_name)
      result.cert_program_url = str(auth.program_url)
      result.cert_signing_id = str(auth.signing_cert_id)

      try:
        # This fills in auth.cert_chain_head. We ignore Asn1Error because
        # we want to extract as much data as possible, no matter if the
        # certificate has expired or not.
        auth.ValidateCertChains(time.gmtime())
      except auth_data.Asn1Error:
        pass
      result.cert_chain_head_issuer = str(auth.cert_chain_head[2])

      if auth.has_countersignature:
        result.cert_countersignature_chain_head_issuer = str(
            auth.counter_chain_head[2])

      certs = []
      for (issuer, serial), cert in iteritems(auth.certificates):
        subject = cert[0][0]["subject"]
        subject_dn = str(dn.DistinguishedName.TraverseRdn(subject[0]))
        not_before = cert[0][0]["validity"]["notBefore"]
        not_after = cert[0][0]["validity"]["notAfter"]
        not_before_time = not_before.ToPythonEpochTime()
        not_after_time = not_after.ToPythonEpochTime()
        not_before_time_str = time.asctime(time.gmtime(not_before_time))
        not_after_time_str = time.asctime(time.gmtime(not_after_time))

        certs.append(
            dict(
                issuer=issuer,
                serial=serial,
                subject=subject_dn,
                not_before_time=not_before_time_str,
                not_after_time=not_after_time_str))
      result.cert_certificates = str(certs)

    # Verify_sigs library can basically throw all kinds of exceptions so
    # we have to use broad except here.
    except Exception as e:  # pylint: disable=broad-except
      logging.error(e)

Response: Parses the signed certificate data and updates the result rdfvalue.


Instruction:
How would you explain what the following Python 3 function does
def ParseFileHash(hash_obj, result):
    if hash_obj.HasField("md5"):
      result.hash_md5 = str(hash_obj.md5)

    if hash_obj.HasField("sha1"):
      result.hash_sha1 = str(hash_obj.sha1)

    if hash_obj.HasField("sha256"):
      result.hash_sha256 = str(hash_obj.sha256)

    if hash_obj.HasField("pecoff_md5"):
      result.pecoff_hash_md5 = str(hash_obj.pecoff_md5)

    if hash_obj.HasField("pecoff_sha1"):
      result.pecoff_hash_sha1 = str(hash_obj.pecoff_sha1)

    if hash_obj.HasField("signed_data"):
      StatEntryToExportedFileConverter.ParseSignedData(hash_obj.signed_data[0],
                                                       result)

Response: Parses a Hash rdfvalue into ExportedFile s fields.


Instruction:
Can you generate the documentation for the following Python 3 function
def Convert(self, metadata, stat_entry, token=None):
    return self.BatchConvert([(metadata, stat_entry)], token=token)

Response: Converts a StatEntry to ExportedFile.


Instruction:
Implement a function in Python 3 to
filter out registry keys to operate on files.

Response:
def _RemoveRegistryKeys(self, metadata_value_pairs):
    """Filter out registry keys to operate on files."""
    filtered_pairs = []
    for metadata, stat_entry in metadata_value_pairs:
      # Ignore registry keys.
      if stat_entry.pathspec.pathtype != rdf_paths.PathSpec.PathType.REGISTRY:
        filtered_pairs.append((metadata, stat_entry))

    return filtered_pairs


Instruction:
How would you explain what the following Python 3 function does
def _OpenFilesForRead(self, metadata_value_pairs, token):
    aff4_paths = [
        result.AFF4Path(metadata.client_urn)
        for metadata, result in metadata_value_pairs
    ]
    fds = aff4.FACTORY.MultiOpen(aff4_paths, mode="r", token=token)
    fds_dict = dict([(fd.urn, fd) for fd in fds])
    return fds_dict

Response: Open files for reading.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _ExportFileContent(self, aff4_object, result):
    if self.options.export_files_contents:
      try:
        result.content = aff4_object.Read(self.MAX_CONTENT_SIZE)
        result.content_sha256 = hashlib.sha256(result.content).hexdigest()
      except (IOError, AttributeError) as e:
        logging.warning("Can't read content of %s: %s", aff4_object.urn, e)

Response: Adds file content from aff4_object to result.


Instruction:
Can you write a function in Python 3 where it
converts a batch of StatEntry values to ExportedFile values at once.

Response:
def BatchConvert(self, metadata_value_pairs, token=None):
    """Converts a batch of StatEntry value to ExportedFile values at once.

    Args:
      metadata_value_pairs: a list or a generator of tuples (metadata, value),
        where metadata is ExportedMetadata to be used for conversion and value
        is a StatEntry to be converted.
      token: Security token:

    Yields:
      Resulting ExportedFile values. Empty list is a valid result and means that
      conversion wasn't possible.
    """
    if data_store.RelationalDBEnabled():
      result_generator = self._BatchConvertRelational(metadata_value_pairs)
    else:
      result_generator = self._BatchConvertLegacy(
          metadata_value_pairs, token=token)

    for r in result_generator:
      yield r


Instruction:
Write a Python 3 script to
convert a StatEntry to ExportedRegistryKey.

Response:
def Convert(self, metadata, stat_entry, token=None):
    """Converts StatEntry to ExportedRegistryKey.

    Does nothing if StatEntry corresponds to a file and not a registry entry.

    Args:
      metadata: ExportedMetadata to be used for conversion.
      stat_entry: StatEntry to be converted.
      token: Security token.

    Returns:
      List or generator with resulting RDFValues. Empty list if StatEntry
      corresponds to a file and not to a registry entry.
    """
    if stat_entry.pathspec.pathtype != rdf_paths.PathSpec.PathType.REGISTRY:
      return []

    result = ExportedRegistryKey(
        metadata=metadata,
        urn=stat_entry.AFF4Path(metadata.client_urn),
        last_modified=stat_entry.st_mtime)

    if (stat_entry.HasField("registry_type") and
        stat_entry.HasField("registry_data")):

      result.type = stat_entry.registry_type

      # `data` can be value of arbitrary type and we need to return `bytes`. So,
      # if it is `bytes` we just pass it through. If it is not, we stringify it
      # to some human-readable form and turn it to `bytes` by UTF-8 encoding.
      data = stat_entry.registry_data.GetValue()
      if isinstance(data, bytes):
        result.data = data
      else:
        result.data = str(data).encode("utf-8")

    return [result]


Instruction:
Can you generate the documentation for the following Python 3 function
def Convert(self, metadata, conn, token=None):

    result = ExportedNetworkConnection(
        metadata=metadata,
        family=conn.family,
        type=conn.type,
        local_address=conn.local_address,
        remote_address=conn.remote_address,
        state=conn.state,
        pid=conn.pid,
        ctime=conn.ctime)
    return [result]

Response: Converts a NetworkConnection to ExportedNetworkConnection.


Instruction:
How would you code a function in Python 3 to
convert Process to ExportedProcess.

Response:
def Convert(self, metadata, process, token=None):
    """Converts Process to ExportedProcess."""

    result = ExportedProcess(
        metadata=metadata,
        pid=process.pid,
        ppid=process.ppid,
        name=process.name,
        exe=process.exe,
        cmdline=" ".join(process.cmdline),
        ctime=process.ctime,
        real_uid=process.real_uid,
        effective_uid=process.effective_uid,
        saved_uid=process.saved_uid,
        real_gid=process.real_gid,
        effective_gid=process.effective_gid,
        saved_gid=process.saved_gid,
        username=process.username,
        terminal=process.terminal,
        status=process.status,
        nice=process.nice,
        cwd=process.cwd,
        num_threads=process.num_threads,
        user_cpu_time=process.user_cpu_time,
        system_cpu_time=process.system_cpu_time,
        cpu_percent=process.cpu_percent,
        rss_size=process.RSS_size,
        vms_size=process.VMS_size,
        memory_percent=process.memory_percent)
    return [result]


Instruction:
Given the following Python 3 function, write the documentation
def Convert(self, metadata, process, token=None):

    conn_converter = NetworkConnectionToExportedNetworkConnectionConverter(
        options=self.options)
    return conn_converter.BatchConvert(
        [(metadata, conn) for conn in process.connections], token=token)

Response: Converts Process to ExportedNetworkConnection.


Instruction:
Implement a function in Python 3 to
convert Process to ExportedOpenFile.

Response:
def Convert(self, metadata, process, token=None):
    """Converts Process to ExportedOpenFile."""

    for f in process.open_files:
      yield ExportedOpenFile(metadata=metadata, pid=process.pid, path=f)


Instruction:
Make a summary of the following Python 3 code
def Convert(self, metadata, interface, token=None):
    ip4_addresses = []
    ip6_addresses = []
    for addr in interface.addresses:
      if addr.address_type == addr.Family.INET:
        ip4_addresses.append(addr.human_readable_address)
      elif addr.address_type == addr.Family.INET6:
        ip6_addresses.append(addr.human_readable_address)
      else:
        raise ValueError("Invalid address type: %s" % addr.address_type)

    result = ExportedNetworkInterface(
        metadata=metadata,
        ifname=interface.ifname,
        ip4_addresses=" ".join(ip4_addresses),
        ip6_addresses=" ".join(ip6_addresses))

    if interface.mac_address:
      result.mac_address = interface.mac_address.human_readable_address

    yield result

Response: Converts an Interface to ExportedNetworkInterfaces.


Instruction:
Can you generate the documentation for the following Python 3 function
def Convert(self, metadata, config, token=None):
    result = ExportedDNSClientConfiguration(
        metadata=metadata,
        dns_servers=" ".join(config.dns_server),
        dns_suffixes=" ".join(config.dns_suffix))
    yield result

Response: Converts DNSClientConfiguration to ExportedDNSClientConfiguration.


Instruction:
Can you generate the documentation for the following Python 3 function
def Convert(self, metadata, client_summary, token=None):
    for interface in client_summary.interfaces:
      yield super(ClientSummaryToExportedNetworkInterfaceConverter,
                  self).Convert(
                      metadata, interface, token=token).next()

Response: Converts ClientSummary to ExportedNetworkInterfaces.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _SeparateTypes(self, metadata_value_pairs):
    registry_pairs = []
    file_pairs = []
    match_pairs = []
    for metadata, result in metadata_value_pairs:
      if (result.stat_entry.pathspec.pathtype ==
          rdf_paths.PathSpec.PathType.REGISTRY):
        registry_pairs.append((metadata, result.stat_entry))
      else:
        file_pairs.append((metadata, result))

      match_pairs.extend([(metadata, match) for match in result.matches])

    return registry_pairs, file_pairs, match_pairs

Response: Separate files registry keys grep matches.


Instruction:
Write a Python 3 function for
converting GrrMessage into a set of RDFValues.

Response:
def Convert(self, metadata, grr_message, token=None):
    """Converts GrrMessage into a set of RDFValues.

    Args:
      metadata: ExportedMetadata to be used for conversion.
      grr_message: GrrMessage to be converted.
      token: Security token.

    Returns:
      List or generator with resulting RDFValues.
    """
    return self.BatchConvert([(metadata, grr_message)], token=token)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def BatchConvert(self, metadata_value_pairs, token=None):

    # Group messages by source (i.e. by client urn).
    msg_dict = {}
    for metadata, msg in metadata_value_pairs:
      msg_dict.setdefault(msg.source, []).append((metadata, msg))

    metadata_objects = []
    metadata_to_fetch = []

    # Open the clients we don't have metadata for and fetch metadata.
    for client_urn in msg_dict:
      try:
        metadata_objects.append(self.cached_metadata[client_urn])
      except KeyError:
        metadata_to_fetch.append(client_urn)

    if metadata_to_fetch:
      if data_store.RelationalDBEnabled():
        client_ids = set(urn.Basename() for urn in metadata_to_fetch)
        infos = data_store.REL_DB.MultiReadClientFullInfo(client_ids)

        fetched_metadata = [
            GetMetadata(client_id, info) for client_id, info in infos.items()
        ]
      else:
        client_fds = aff4.FACTORY.MultiOpen(
            metadata_to_fetch, mode="r", token=token)

        fetched_metadata = [
            GetMetadataLegacy(client_fd, token=token)
            for client_fd in client_fds
        ]

      for metadata in fetched_metadata:
        self.cached_metadata[metadata.client_urn] = metadata
      metadata_objects.extend(fetched_metadata)

    data_by_type = {}
    for metadata in metadata_objects:
      try:
        for original_metadata, message in msg_dict[metadata.client_urn]:
          # Get source_urn and annotations from the original metadata
          # provided and original_timestamp from the payload age.
          new_metadata = ExportedMetadata(metadata)
          new_metadata.source_urn = original_metadata.source_urn
          new_metadata.annotations = original_metadata.annotations
          new_metadata.original_timestamp = message.payload.age
          cls_name = message.payload.__class__.__name__

          # Create a dict of values for conversion keyed by type, so we can
          # apply the right converters to the right object types
          if cls_name not in data_by_type:
            converters_classes = ExportConverter.GetConvertersByValue(
                message.payload)
            data_by_type[cls_name] = {
                "converters": [cls(self.options) for cls in converters_classes],
                "batch_data": [(new_metadata, message.payload)]
            }
          else:
            data_by_type[cls_name]["batch_data"].append(
                (new_metadata, message.payload))

      except KeyError:
        pass

    # Run all converters against all objects of the relevant type
    converted_batch = []
    for dataset in itervalues(data_by_type):
      for converter in dataset["converters"]:
        converted_batch.extend(
            converter.BatchConvert(dataset["batch_data"], token=token))

    return converted_batch

Response: Converts a batch of GrrMessages into a set of RDFValues at once.


Instruction:
Make a summary of the following Python 3 code
def BatchConvert(self, metadata_value_pairs, token=None):

    urns = [urn for metadata, urn in metadata_value_pairs]
    urns_dict = dict((urn, metadata) for metadata, urn in metadata_value_pairs)

    results = []
    for hash_urn, client_files in filestore.HashFileStore.GetClientsForHashes(
        urns, token=token):
      for hit in client_files:
        metadata = ExportedMetadata(urns_dict[hash_urn])
        metadata.client_urn = rdfvalue.RDFURN(hit).Split(2)[0]

        result = ExportedFileStoreHash(
            metadata=metadata,
            hash=hash_urn.hash_value,
            fingerprint_type=hash_urn.fingerprint_type,
            hash_type=hash_urn.hash_type,
            target_urn=hit)
        results.append(result)

    return results

Response: Convert batch of FileStoreHashs.


Instruction:
Given the following Python 3 function, write the documentation
def Convert(self, metadata, checkresult, token=None):
    if checkresult.HasField("anomaly"):
      for anomaly in checkresult.anomaly:
        exported_anomaly = ExportedAnomaly(
            type=anomaly.type,
            severity=anomaly.severity,
            confidence=anomaly.confidence)
        if anomaly.symptom:
          exported_anomaly.symptom = anomaly.symptom
        if anomaly.explanation:
          exported_anomaly.explanation = anomaly.explanation
        if anomaly.generated_by:
          exported_anomaly.generated_by = anomaly.generated_by
        if anomaly.anomaly_reference_id:
          exported_anomaly.anomaly_reference_id = "\n".join(
              anomaly.anomaly_reference_id)
        if anomaly.finding:
          exported_anomaly.finding = "\n".join(anomaly.finding)
        yield ExportedCheckResult(
            metadata=metadata,
            check_id=checkresult.check_id,
            anomaly=exported_anomaly)
    else:
      yield ExportedCheckResult(
          metadata=metadata, check_id=checkresult.check_id)

Response: Converts a single CheckResult into a list of ExportedCheckResult objects.


Instruction:
Write a Python 3 script for
converting original result via given converter.

Response:
def GetExportedResult(self,
                        original_result,
                        converter,
                        metadata=None,
                        token=None):
    """Converts original result via given converter.."""

    exported_results = list(
        converter.Convert(
            metadata or ExportedMetadata(), original_result, token=token))

    if not exported_results:
      raise ExportError("Got 0 exported result when a single one "
                        "was expected.")

    if len(exported_results) > 1:
      raise ExportError("Got > 1 exported results when a single "
                        "one was expected, seems like a logical bug.")

    return exported_results[0]


Instruction:
Can you write a function in Python 3 where it
checks if given RDFValue is a registry StatEntry.

Response:
def IsRegistryStatEntry(self, original_result):
    """Checks if given RDFValue is a registry StatEntry."""
    return (original_result.pathspec.pathtype ==
            rdf_paths.PathSpec.PathType.REGISTRY)


Instruction:
Implement a function in Python 3 to
check if given RDFValue is a file StatEntry.

Response:
def IsFileStatEntry(self, original_result):
    """Checks if given RDFValue is a file StatEntry."""
    return (original_result.pathspec.pathtype in [
        rdf_paths.PathSpec.PathType.OS, rdf_paths.PathSpec.PathType.TSK
    ])


Instruction:
How would you explain what the following Python 3 function does
def Convert(self, metadata, value, token=None):

    for r in self.BatchConvert([(metadata, value)], token=token):
      yield r

Response: Converts a single ArtifactFilesDownloaderResult.


Instruction:
Write a Python 3 function that can
convert a single YaraProcessScanMatch.

Response:
def Convert(self, metadata, yara_match, token=None):
    """Convert a single YaraProcessScanMatch."""

    conv = ProcessToExportedProcessConverter(options=self.options)
    process = list(
        conv.Convert(ExportedMetadata(), yara_match.process, token=token))[0]

    seen_rules = set()
    for m in yara_match.match:
      if m.rule_name in seen_rules:
        continue

      seen_rules.add(m.rule_name)
      yield ExportedYaraProcessScanMatch(
          metadata=metadata,
          process=process,
          rule_name=m.rule_name,
          scan_time_us=yara_match.scan_time_us)


Instruction:
Implement a Python 3 function for
creating a dynamic RDF proto struct class for given osquery table.

Response:
def _RDFClass(cls, table):
    """Creates a dynamic RDF proto struct class for given osquery table.

    The fields of the proto will correspond to the columns of the table.

    Args:
      table: An osquery table for which the class is about to be generated.

    Returns:
      A class object corresponding to the given table.
    """
    rdf_cls_name = "OsqueryTable{}".format(hash(table.query))
    try:
      return cls._rdf_cls_cache[rdf_cls_name]
    except KeyError:
      pass

    rdf_cls = compatibility.MakeType(rdf_cls_name,
                                     (rdf_structs.RDFProtoStruct,), {})

    rdf_cls.AddDescriptor(
        rdf_structs.ProtoEmbedded(
            name="metadata", field_number=1, nested=ExportedMetadata))

    rdf_cls.AddDescriptor(
        rdf_structs.ProtoString(name="__query__", field_number=2))

    for idx, column in enumerate(table.header.columns):
      # It is possible that RDF column is named "metadata". To avoid name clash
      # we must rename it to `__metadata__`.
      if column.name == "metadata":
        name = "__metadata__"
      else:
        name = column.name

      descriptor = rdf_structs.ProtoString(name=name, field_number=idx + 3)
      rdf_cls.AddDescriptor(descriptor)

    cls._rdf_cls_cache[rdf_cls_name] = rdf_cls
    return rdf_cls


Instruction:
How would you implement a function in Python 3 that
extracts the path from a client action response.

Response:
def _ExtractPath(response, pathspec_attribute=None):
  """Returns the path from a client action response as a string.

  Args:
    response: A client action response.
    pathspec_attribute: Specifies the field which stores the pathspec.

  Returns:
    The path as a string or None if no path is found.

  """
  path_specification = response

  if pathspec_attribute is not None:
    if response.HasField(pathspec_attribute):
      path_specification = response.Get(pathspec_attribute)

  if path_specification.HasField("pathspec"):
    path_specification = path_specification.pathspec

  if path_specification.HasField("path"):
    path_specification = path_specification.path

  if isinstance(path_specification, Text):
    return path_specification
  return None


Instruction:
Write a Python 3 script for
returning an rdf_artifacts. CollectedArtifact rdf object for the requested artifact.

Response:
def _CollectArtifact(self, artifact, apply_parsers):
    """Returns an `CollectedArtifact` rdf object for the requested artifact."""
    artifact_result = rdf_artifacts.CollectedArtifact(name=artifact.name)

    if apply_parsers:
      parser_factory = parsers.ArtifactParserFactory(str(artifact.name))
    else:
      parser_factory = None

    for source_result_list in self._ProcessSources(artifact.sources,
                                                   parser_factory):
      for response in source_result_list:
        action_result = rdf_artifacts.ClientActionResult()
        action_result.type = response.__class__.__name__
        action_result.value = response
        artifact_result.action_results.append(action_result)
        self.UpdateKnowledgeBase(response, artifact.provides)

    return artifact_result


Instruction:
Given the following Python 3 function, write the documentation
def UpdateKnowledgeBase(self, response, provides):

    if isinstance(response, rdf_anomaly.Anomaly):
      return

    if isinstance(response, rdf_client.User):
      self.knowledge_base.MergeOrAddUser(response)
      return

    if isinstance(response, rdf_protodict.Dict):
      response_dict = response.ToDict()
      for attribute, value in iteritems(response_dict):
        if attribute in provides:
          self.SetKnowledgeBaseValue(attribute, value)
      return

    # If its not a dict we only support a single value.
    if len(provides) == 1:
      self.SetKnowledgeBaseValue(provides[0], response)

Response: Updates the values in the knowledge base based on responses.


Instruction:
Make a summary of the following Python 3 code
def _ProcessSources(self, sources, parser_factory):
    for source in sources:
      for action, request in self._ParseSourceType(source):
        yield self._RunClientAction(action, request, parser_factory,
                                    source.path_type)

Response: Iterates through sources yielding action responses.


Instruction:
Can you implement a function in Python 3 that
runs the client action with the request and parses the result.

Response:
def _RunClientAction(self, action, request, parser_factory, path_type):
    """Runs the client action  with the request and parses the result."""
    responses = list(action(request))

    if parser_factory is None:
      return responses

    # parse the responses
    parsed_responses = []

    for response in responses:
      for parser in parser_factory.SingleResponseParsers():
        parsed_responses.extend(
            parser.ParseResponse(self.knowledge_base, response, path_type))

      for parser in parser_factory.SingleFileParsers():
        precondition.AssertType(response, rdf_client_fs.StatEntry)
        pathspec = response.pathspec
        with vfs.VFSOpen(pathspec) as filedesc:
          parsed_responses.extend(
              parser.ParseFile(self.knowledge_base, pathspec, filedesc))

    for parser in parser_factory.MultiResponseParsers():
      parsed_responses.extend(
          parser.ParseResponses(self.knowledge_base, responses))

    for parser in parser_factory.MultiFileParsers():
      precondition.AssertIterableType(responses, rdf_client_fs.StatEntry)
      pathspecs = [response.pathspec for response in responses]
      with vfs.VFSMultiOpen(pathspecs) as filedescs:
        parsed_responses.extend(
            parser.ParseFiles(self.knowledge_base, pathspecs, filedescs))

    return parsed_responses


Instruction:
Explain what the following Python 3 code does
def _ParseSourceType(self, source):
    type_name = rdf_artifacts.ArtifactSource.SourceType
    switch = {
        type_name.COMMAND: self._ProcessCommandSource,
        type_name.DIRECTORY: self._ProcessFileSource,
        type_name.FILE: self._ProcessFileSource,
        type_name.GREP: self._ProcessGrepSource,
        type_name.REGISTRY_KEY: self._ProcessRegistryKeySource,
        type_name.REGISTRY_VALUE: self._ProcessRegistryValueSource,
        type_name.WMI: self._ProcessWmiSource,
        type_name.ARTIFACT_FILES: self._ProcessArtifactFilesSource,
        type_name.GRR_CLIENT_ACTION: self._ProcessClientActionSource
    }
    source_type = source.base_source.type

    try:
      source_type_action = switch[source_type]
    except KeyError:
      raise ValueError("Incorrect source type: %s" % source_type)

    for res in source_type_action(source):
      yield res

Response: Parses the source type and yields the results.


Instruction:
Explain what the following Python 3 code does
def _ProcessRegistryKeySource(self, source):
    keys = source.base_source.attributes.get("keys", [])
    if not keys:
      return

    interpolated_paths = artifact_utils.InterpolateListKbAttributes(
        input_list=keys,
        knowledge_base=self.knowledge_base,
        ignore_errors=self.ignore_interpolation_errors)

    glob_expressions = map(rdf_paths.GlobExpression, interpolated_paths)

    patterns = []
    for pattern in glob_expressions:
      patterns.extend(pattern.Interpolate(knowledge_base=self.knowledge_base))
    patterns.sort(key=len, reverse=True)

    file_finder_action = rdf_file_finder.FileFinderAction.Stat()
    request = rdf_file_finder.FileFinderArgs(
        paths=patterns,
        action=file_finder_action,
        follow_links=True,
        pathtype=rdf_paths.PathSpec.PathType.REGISTRY)
    action = vfs_file_finder.RegistryKeyFromClient

    yield action, request

Response: Process a registry key source.


Instruction:
Can you tell what is the following Python 3 function doing
def _ProcessGrepSource(self, source):
    attributes = source.base_source.attributes
    paths = artifact_utils.InterpolateListKbAttributes(
        attributes["paths"], self.knowledge_base,
        self.ignore_interpolation_errors)
    regex = utils.RegexListDisjunction(attributes["content_regex_list"])
    condition = rdf_file_finder.FileFinderCondition.ContentsRegexMatch(
        regex=regex, mode="ALL_HITS")
    file_finder_action = rdf_file_finder.FileFinderAction.Stat()
    request = rdf_file_finder.FileFinderArgs(
        paths=paths,
        action=file_finder_action,
        conditions=[condition],
        follow_links=True)
    action = file_finder.FileFinderOSFromClient

    yield action, request

Response: Yields a generator that yields a file finder action and request.


Instruction:
Write a Python 3 function for
getting artifact responses extract paths and send corresponding files.

Response:
def _ProcessArtifactFilesSource(self, source):
    """Get artifact responses, extract paths and send corresponding files."""

    if source.path_type != rdf_paths.PathSpec.PathType.OS:
      raise ValueError("Only supported path type is OS.")

    # TODO(user): Check paths for GlobExpressions.
    # If it contains a * then FileFinder will interpret it as GlobExpression and
    # expand it. FileFinderArgs needs an option to treat paths literally.

    paths = []
    pathspec_attribute = source.base_source.attributes.get("pathspec_attribute")

    for source_result_list in self._ProcessSources(
        source.artifact_sources, parser_factory=None):
      for response in source_result_list:
        path = _ExtractPath(response, pathspec_attribute)
        if path is not None:
          paths.append(path)

    file_finder_action = rdf_file_finder.FileFinderAction.Download()
    request = rdf_file_finder.FileFinderArgs(
        paths=paths, pathtype=source.path_type, action=file_finder_action)
    action = file_finder.FileFinderOSFromClient

    yield action, request


Instruction:
Write a Python 3 script to
process a file source.

Response:
def _ProcessFileSource(self, source):
    """Glob paths and return StatEntry objects."""

    if source.path_type != rdf_paths.PathSpec.PathType.OS:
      raise ValueError("Only supported path type is OS.")

    paths = artifact_utils.InterpolateListKbAttributes(
        source.base_source.attributes["paths"], self.knowledge_base,
        self.ignore_interpolation_errors)

    file_finder_action = rdf_file_finder.FileFinderAction.Stat()
    request = rdf_file_finder.FileFinderArgs(
        paths=paths, pathtype=source.path_type, action=file_finder_action)
    action = file_finder.FileFinderOSFromClient

    yield action, request


Instruction:
Can you write a function in Python 3 where it
processes a command source.

Response:
def _ProcessCommandSource(self, source):
    """Prepare a request for calling the execute command action."""
    action = standard.ExecuteCommandFromClient
    request = rdf_client_action.ExecuteRequest(
        cmd=source.base_source.attributes["cmd"],
        args=source.base_source.attributes["args"],
    )
    yield action, request


Instruction:
Create a Python 3 function to
write metadata about a client.

Response:
def WriteClientMetadata(self,
                          client_id,
                          certificate=None,
                          fleetspeak_enabled=None,
                          first_seen=None,
                          last_ping=None,
                          last_clock=None,
                          last_ip=None,
                          last_foreman=None,
                          cursor=None):
    """Write metadata about the client."""
    placeholders = []
    values = collections.OrderedDict()

    placeholders.append("%(client_id)s")
    values["client_id"] = db_utils.ClientIDToInt(client_id)

    if certificate:
      placeholders.append("%(certificate)s")
      values["certificate"] = certificate.SerializeToString()
    if fleetspeak_enabled is not None:
      placeholders.append("%(fleetspeak_enabled)s")
      values["fleetspeak_enabled"] = fleetspeak_enabled
    if first_seen is not None:
      placeholders.append("FROM_UNIXTIME(%(first_seen)s)")
      values["first_seen"] = mysql_utils.RDFDatetimeToTimestamp(first_seen)
    if last_ping is not None:
      placeholders.append("FROM_UNIXTIME(%(last_ping)s)")
      values["last_ping"] = mysql_utils.RDFDatetimeToTimestamp(last_ping)
    if last_clock:
      placeholders.append("FROM_UNIXTIME(%(last_clock)s)")
      values["last_clock"] = mysql_utils.RDFDatetimeToTimestamp(last_clock)
    if last_ip:
      placeholders.append("%(last_ip)s")
      values["last_ip"] = last_ip.SerializeToString()
    if last_foreman:
      placeholders.append("FROM_UNIXTIME(%(last_foreman)s)")
      values["last_foreman"] = mysql_utils.RDFDatetimeToTimestamp(last_foreman)

    updates = []
    for column in iterkeys(values):
      updates.append("{column} = VALUES({column})".format(column=column))

    query = """
    INSERT INTO clients ({columns})
    VALUES ({placeholders})
    ON DUPLICATE KEY UPDATE {updates}
    """.format(
        columns=", ".join(iterkeys(values)),
        placeholders=", ".join(placeholders),
        updates=", ".join(updates))

    cursor.execute(query, values)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def MultiReadClientMetadata(self, client_ids, cursor=None):
    ids = [db_utils.ClientIDToInt(client_id) for client_id in client_ids]
    query = ("SELECT client_id, fleetspeak_enabled, certificate, "
             "UNIX_TIMESTAMP(last_ping), "
             "UNIX_TIMESTAMP(last_clock), last_ip, "
             "UNIX_TIMESTAMP(last_foreman), UNIX_TIMESTAMP(first_seen), "
             "UNIX_TIMESTAMP(last_crash_timestamp), "
             "UNIX_TIMESTAMP(last_startup_timestamp) FROM "
             "clients WHERE client_id IN ({})").format(", ".join(["%s"] *
                                                                 len(ids)))
    ret = {}
    cursor.execute(query, ids)
    while True:
      row = cursor.fetchone()
      if not row:
        break
      cid, fs, crt, ping, clk, ip, foreman, first, lct, lst = row
      ret[db_utils.IntToClientID(cid)] = rdf_objects.ClientMetadata(
          certificate=crt,
          fleetspeak_enabled=fs,
          first_seen=mysql_utils.TimestampToRDFDatetime(first),
          ping=mysql_utils.TimestampToRDFDatetime(ping),
          clock=mysql_utils.TimestampToRDFDatetime(clk),
          ip=mysql_utils.StringToRDFProto(rdf_client_network.NetworkAddress,
                                          ip),
          last_foreman_time=mysql_utils.TimestampToRDFDatetime(foreman),
          startup_info_timestamp=mysql_utils.TimestampToRDFDatetime(lst),
          last_crash_timestamp=mysql_utils.TimestampToRDFDatetime(lct))
    return ret

Response: Reads ClientMetadata records for a list of clients.


Instruction:
Explain what the following Python 3 code does
def WriteClientSnapshot(self, snapshot, cursor=None):
    insert_history_query = (
        "INSERT INTO client_snapshot_history(client_id, timestamp, "
        "client_snapshot) VALUES (%s, FROM_UNIXTIME(%s), %s)")
    insert_startup_query = (
        "INSERT INTO client_startup_history(client_id, timestamp, "
        "startup_info) VALUES(%s, FROM_UNIXTIME(%s), %s)")

    now = rdfvalue.RDFDatetime.Now()

    client_platform = snapshot.knowledge_base.os
    current_timestamp = mysql_utils.RDFDatetimeToTimestamp(now)
    client_info = {
        "last_snapshot_timestamp": current_timestamp,
        "last_startup_timestamp": current_timestamp,
        "last_version_string": snapshot.GetGRRVersionString(),
        "last_platform_release": snapshot.Uname(),
    }
    update_clauses = [
        "last_snapshot_timestamp = FROM_UNIXTIME(%(last_snapshot_timestamp)s)",
        "last_startup_timestamp = FROM_UNIXTIME(%(last_startup_timestamp)s)",
        "last_version_string = %(last_version_string)s",
        "last_platform_release = %(last_platform_release)s",
    ]
    if client_platform:
      client_info["last_platform"] = client_platform
      update_clauses.append("last_platform = %(last_platform)s")

    update_query = (
        "UPDATE clients SET {} WHERE client_id = %(client_id)s".format(
            ", ".join(update_clauses)))

    int_client_id = db_utils.ClientIDToInt(snapshot.client_id)
    client_info["client_id"] = int_client_id

    startup_info = snapshot.startup_info
    snapshot.startup_info = None
    try:
      cursor.execute(
          insert_history_query,
          (int_client_id, current_timestamp, snapshot.SerializeToString()))
      cursor.execute(
          insert_startup_query,
          (int_client_id, current_timestamp, startup_info.SerializeToString()))
      cursor.execute(update_query, client_info)
    except MySQLdb.IntegrityError as e:
      raise db.UnknownClientError(snapshot.client_id, cause=e)
    finally:
      snapshot.startup_info = startup_info

Response: Writes a new client snapshot.


Instruction:
Implement a function in Python 3 to
read the latest client snapshots for a list of clients.

Response:
def MultiReadClientSnapshot(self, client_ids, cursor=None):
    """Reads the latest client snapshots for a list of clients."""
    int_ids = [db_utils.ClientIDToInt(cid) for cid in client_ids]
    query = (
        "SELECT h.client_id, h.client_snapshot, UNIX_TIMESTAMP(h.timestamp),"
        "       s.startup_info "
        "FROM clients as c FORCE INDEX (PRIMARY), "
        "client_snapshot_history as h FORCE INDEX (PRIMARY), "
        "client_startup_history as s FORCE INDEX (PRIMARY) "
        "WHERE h.client_id = c.client_id "
        "AND s.client_id = c.client_id "
        "AND h.timestamp = c.last_snapshot_timestamp "
        "AND s.timestamp = c.last_startup_timestamp "
        "AND c.client_id IN ({})").format(", ".join(["%s"] * len(client_ids)))
    ret = {cid: None for cid in client_ids}
    cursor.execute(query, int_ids)

    while True:
      row = cursor.fetchone()
      if not row:
        break
      cid, snapshot, timestamp, startup_info = row
      client_obj = mysql_utils.StringToRDFProto(rdf_objects.ClientSnapshot,
                                                snapshot)
      client_obj.startup_info = mysql_utils.StringToRDFProto(
          rdf_client.StartupInfo, startup_info)
      client_obj.timestamp = mysql_utils.TimestampToRDFDatetime(timestamp)
      ret[db_utils.IntToClientID(cid)] = client_obj
    return ret


Instruction:
Implement a Python 3 function for
reading the full history for a particular client.

Response:
def ReadClientSnapshotHistory(self, client_id, timerange=None, cursor=None):
    """Reads the full history for a particular client."""

    client_id_int = db_utils.ClientIDToInt(client_id)

    query = ("SELECT sn.client_snapshot, st.startup_info, "
             "       UNIX_TIMESTAMP(sn.timestamp) FROM "
             "client_snapshot_history AS sn, "
             "client_startup_history AS st WHERE "
             "sn.client_id = st.client_id AND "
             "sn.timestamp = st.timestamp AND "
             "sn.client_id=%s ")

    args = [client_id_int]
    if timerange:
      time_from, time_to = timerange  # pylint: disable=unpacking-non-sequence

      if time_from is not None:
        query += "AND sn.timestamp >= FROM_UNIXTIME(%s) "
        args.append(mysql_utils.RDFDatetimeToTimestamp(time_from))

      if time_to is not None:
        query += "AND sn.timestamp <= FROM_UNIXTIME(%s) "
        args.append(mysql_utils.RDFDatetimeToTimestamp(time_to))

    query += "ORDER BY sn.timestamp DESC"

    ret = []
    cursor.execute(query, args)
    for snapshot, startup_info, timestamp in cursor.fetchall():
      client = rdf_objects.ClientSnapshot.FromSerializedString(snapshot)
      client.startup_info = rdf_client.StartupInfo.FromSerializedString(
          startup_info)
      client.timestamp = mysql_utils.TimestampToRDFDatetime(timestamp)

      ret.append(client)
    return ret


Instruction:
How would you implement a function in Python 3 that
writes the full history for a particular client.

Response:
def WriteClientSnapshotHistory(self, clients, cursor=None):
    """Writes the full history for a particular client."""
    client_id = clients[0].client_id
    latest_timestamp = max(client.timestamp for client in clients)

    query = ""
    params = {
        "client_id": db_utils.ClientIDToInt(client_id),
        "latest_timestamp": mysql_utils.RDFDatetimeToTimestamp(latest_timestamp)
    }

    for idx, client in enumerate(clients):
      startup_info = client.startup_info
      client.startup_info = None

      query += """
      INSERT INTO client_snapshot_history (client_id, timestamp,
                                           client_snapshot)
      VALUES (%(client_id)s, FROM_UNIXTIME(%(timestamp_{idx})s),
              %(client_snapshot_{idx})s);

      INSERT INTO client_startup_history (client_id, timestamp,
                                          startup_info)
      VALUES (%(client_id)s, FROM_UNIXTIME(%(timestamp_{idx})s),
              %(startup_info_{idx})s);
      """.format(idx=idx)

      params.update({
          "timestamp_{idx}".format(idx=idx):
              mysql_utils.RDFDatetimeToTimestamp(client.timestamp),
          "client_snapshot_{idx}".format(idx=idx):
              client.SerializeToString(),
          "startup_info_{idx}".format(idx=idx):
              startup_info.SerializeToString(),
      })

      client.startup_info = startup_info

    query += """
    UPDATE clients
       SET last_snapshot_timestamp = FROM_UNIXTIME(%(latest_timestamp)s)
     WHERE client_id = %(client_id)s
       AND (last_snapshot_timestamp IS NULL OR
            last_snapshot_timestamp < FROM_UNIXTIME(%(latest_timestamp)s));

    UPDATE clients
       SET last_startup_timestamp = FROM_UNIXTIME(%(latest_timestamp)s)
     WHERE client_id = %(client_id)s
       AND (last_startup_timestamp IS NULL OR
            last_startup_timestamp < FROM_UNIXTIME(%(latest_timestamp)s));
    """

    try:
      cursor.execute(query, params)
    except MySQLdb.IntegrityError as error:
      raise db.UnknownClientError(client_id, cause=error)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def WriteClientStartupInfo(self, client_id, startup_info, cursor=None):
    query = """
    SET @now = NOW(6);

    INSERT INTO client_startup_history (client_id, timestamp, startup_info)
         VALUES (%(client_id)s, @now, %(startup_info)s);

    UPDATE clients
       SET last_startup_timestamp = @now
     WHERE client_id = %(client_id)s;
    """

    params = {
        "client_id": db_utils.ClientIDToInt(client_id),
        "startup_info": startup_info.SerializeToString(),
    }

    try:
      cursor.execute(query, params)
    except MySQLdb.IntegrityError as e:
      raise db.UnknownClientError(client_id, cause=e)

Response: Writes a new client startup record.


Instruction:
How would you explain what the following Python 3 function does
def ReadClientStartupInfo(self, client_id, cursor=None):
    query = (
        "SELECT startup_info, UNIX_TIMESTAMP(timestamp) "
        "FROM clients, client_startup_history "
        "WHERE clients.last_startup_timestamp=client_startup_history.timestamp "
        "AND clients.client_id=client_startup_history.client_id "
        "AND clients.client_id=%s")
    cursor.execute(query, [db_utils.ClientIDToInt(client_id)])
    row = cursor.fetchone()
    if row is None:
      return None

    startup_info, timestamp = row
    res = rdf_client.StartupInfo.FromSerializedString(startup_info)
    res.timestamp = mysql_utils.TimestampToRDFDatetime(timestamp)
    return res

Response: Reads the latest client startup record for a single client.


Instruction:
Can you write a function in Python 3 where it
reads the full startup history for a particular client.

Response:
def ReadClientStartupInfoHistory(self, client_id, timerange=None,
                                   cursor=None):
    """Reads the full startup history for a particular client."""

    client_id_int = db_utils.ClientIDToInt(client_id)

    query = ("SELECT startup_info, UNIX_TIMESTAMP(timestamp) "
             "FROM client_startup_history "
             "WHERE client_id=%s ")
    args = [client_id_int]

    if timerange:
      time_from, time_to = timerange  # pylint: disable=unpacking-non-sequence

      if time_from is not None:
        query += "AND timestamp >= FROM_UNIXTIME(%s) "
        args.append(mysql_utils.RDFDatetimeToTimestamp(time_from))

      if time_to is not None:
        query += "AND timestamp <= FROM_UNIXTIME(%s) "
        args.append(mysql_utils.RDFDatetimeToTimestamp(time_to))

    query += "ORDER BY timestamp DESC "

    ret = []
    cursor.execute(query, args)

    for startup_info, timestamp in cursor.fetchall():
      si = rdf_client.StartupInfo.FromSerializedString(startup_info)
      si.timestamp = mysql_utils.TimestampToRDFDatetime(timestamp)
      ret.append(si)
    return ret


Instruction:
Can you implement a function in Python 3 that
converts a database response to a list of ClientFullInfo objects.

Response:
def _ResponseToClientsFullInfo(self, response):
    """Creates a ClientFullInfo object from a database response."""
    c_full_info = None
    prev_cid = None
    for row in response:
      (cid, fs, crt, ping, clk, ip, foreman, first, last_client_ts,
       last_crash_ts, last_startup_ts, client_obj, client_startup_obj,
       last_startup_obj, label_owner, label_name) = row

      if cid != prev_cid:
        if c_full_info:
          yield db_utils.IntToClientID(prev_cid), c_full_info

        metadata = rdf_objects.ClientMetadata(
            certificate=crt,
            fleetspeak_enabled=fs,
            first_seen=mysql_utils.TimestampToRDFDatetime(first),
            ping=mysql_utils.TimestampToRDFDatetime(ping),
            clock=mysql_utils.TimestampToRDFDatetime(clk),
            ip=mysql_utils.StringToRDFProto(rdf_client_network.NetworkAddress,
                                            ip),
            last_foreman_time=mysql_utils.TimestampToRDFDatetime(foreman),
            startup_info_timestamp=mysql_utils.TimestampToRDFDatetime(
                last_startup_ts),
            last_crash_timestamp=mysql_utils.TimestampToRDFDatetime(
                last_crash_ts))

        if client_obj is not None:
          l_snapshot = rdf_objects.ClientSnapshot.FromSerializedString(
              client_obj)
          l_snapshot.timestamp = mysql_utils.TimestampToRDFDatetime(
              last_client_ts)
          l_snapshot.startup_info = rdf_client.StartupInfo.FromSerializedString(
              client_startup_obj)
          l_snapshot.startup_info.timestamp = l_snapshot.timestamp
        else:
          l_snapshot = rdf_objects.ClientSnapshot(
              client_id=db_utils.IntToClientID(cid))

        if last_startup_obj is not None:
          startup_info = rdf_client.StartupInfo.FromSerializedString(
              last_startup_obj)
          startup_info.timestamp = mysql_utils.TimestampToRDFDatetime(
              last_startup_ts)
        else:
          startup_info = None

        prev_cid = cid
        c_full_info = rdf_objects.ClientFullInfo(
            metadata=metadata,
            labels=[],
            last_snapshot=l_snapshot,
            last_startup_info=startup_info)

      if label_owner and label_name:
        c_full_info.labels.append(
            rdf_objects.ClientLabel(name=label_name, owner=label_owner))

    if c_full_info:
      yield db_utils.IntToClientID(prev_cid), c_full_info


Instruction:
Can you generate the documentation for the following Python 3 function
def MultiReadClientFullInfo(self, client_ids, min_last_ping=None,
                              cursor=None):
    """Reads full client information for a list of clients."""
    if not client_ids:
      return {}

    query = (
        "SELECT "
        "c.client_id, c.fleetspeak_enabled, c.certificate, "
        "UNIX_TIMESTAMP(c.last_ping), UNIX_TIMESTAMP(c.last_clock), "
        "c.last_ip, UNIX_TIMESTAMP(c.last_foreman), "
        "UNIX_TIMESTAMP(c.first_seen), "
        "UNIX_TIMESTAMP(c.last_snapshot_timestamp), "
        "UNIX_TIMESTAMP(c.last_crash_timestamp), "
        "UNIX_TIMESTAMP(c.last_startup_timestamp), "
        "h.client_snapshot, s.startup_info, s_last.startup_info, "
        "l.owner_username, l.label "
        "FROM clients as c "
        "FORCE INDEX (PRIMARY) "
        "LEFT JOIN client_snapshot_history as h FORCE INDEX (PRIMARY) ON ( "
        "c.client_id = h.client_id AND "
        "h.timestamp = c.last_snapshot_timestamp) "
        "LEFT JOIN client_startup_history as s FORCE INDEX (PRIMARY) ON ( "
        "c.client_id = s.client_id AND "
        "s.timestamp = c.last_snapshot_timestamp) "
        "LEFT JOIN client_startup_history as s_last FORCE INDEX (PRIMARY) ON ( "
        "c.client_id = s_last.client_id "
        "AND s_last.timestamp = c.last_startup_timestamp) "
        "LEFT JOIN client_labels AS l FORCE INDEX (PRIMARY) "
        "ON (c.client_id = l.client_id) ")

    query += "WHERE c.client_id IN (%s) " % ", ".join(["%s"] * len(client_ids))

    values = [db_utils.ClientIDToInt(cid) for cid in client_ids]
    if min_last_ping is not None:
      query += "AND c.last_ping >= FROM_UNIXTIME(%s)"
      values.append(mysql_utils.RDFDatetimeToTimestamp(min_last_ping))

    cursor.execute(query, values)
    return dict(self._ResponseToClientsFullInfo(cursor.fetchall()))

Response: Reads full client information for a list of clients.


Instruction:
Can you generate the documentation for the following Python 3 function
def ReadClientLastPings(self,
                          min_last_ping=None,
                          max_last_ping=None,
                          fleetspeak_enabled=None,
                          cursor=None):
    """Reads client ids for all clients in the database."""
    query = "SELECT client_id, UNIX_TIMESTAMP(last_ping) FROM clients "
    query_values = []
    where_filters = []
    if min_last_ping is not None:
      where_filters.append("last_ping >= FROM_UNIXTIME(%s) ")
      query_values.append(mysql_utils.RDFDatetimeToTimestamp(min_last_ping))
    if max_last_ping is not None:
      where_filters.append(
          "(last_ping IS NULL OR last_ping <= FROM_UNIXTIME(%s))")
      query_values.append(mysql_utils.RDFDatetimeToTimestamp(max_last_ping))
    if fleetspeak_enabled is not None:
      if fleetspeak_enabled:
        where_filters.append("fleetspeak_enabled IS TRUE")
      else:
        where_filters.append(
            "(fleetspeak_enabled IS NULL OR fleetspeak_enabled IS FALSE)")

    if where_filters:
      query += "WHERE " + "AND ".join(where_filters)
    cursor.execute(query, query_values)
    last_pings = {}
    for int_client_id, last_ping in cursor.fetchall():
      client_id = db_utils.IntToClientID(int_client_id)
      last_pings[client_id] = mysql_utils.TimestampToRDFDatetime(last_ping)
    return last_pings

Response: Reads the last ping timestamps for all clients in the database.


Instruction:
Create a Python 3 function to
associate the provided keywords with the client.

Response:
def AddClientKeywords(self, client_id, keywords, cursor=None):
    """Associates the provided keywords with the client."""
    cid = db_utils.ClientIDToInt(client_id)
    keywords = set(keywords)
    args = [(cid, mysql_utils.Hash(kw), kw) for kw in keywords]
    args = list(collection.Flatten(args))

    query = """
        INSERT INTO client_keywords (client_id, keyword_hash, keyword)
        VALUES {}
        ON DUPLICATE KEY UPDATE timestamp = NOW(6)
            """.format(", ".join(["(%s, %s, %s)"] * len(keywords)))
    try:
      cursor.execute(query, args)
    except MySQLdb.IntegrityError as e:
      raise db.UnknownClientError(client_id, cause=e)


Instruction:
Implement a Python 3 function for
removing the association of a particular client to a keyword.

Response:
def RemoveClientKeyword(self, client_id, keyword, cursor=None):
    """Removes the association of a particular client to a keyword."""
    cursor.execute(
        "DELETE FROM client_keywords "
        "WHERE client_id = %s AND keyword_hash = %s",
        [db_utils.ClientIDToInt(client_id),
         mysql_utils.Hash(keyword)])


Instruction:
Explain what the following Python 3 code does
def ListClientsForKeywords(self, keywords, start_time=None, cursor=None):
    keywords = set(keywords)
    hash_to_kw = {mysql_utils.Hash(kw): kw for kw in keywords}
    result = {kw: [] for kw in keywords}

    query = """
      SELECT keyword_hash, client_id
      FROM client_keywords
      FORCE INDEX (client_index_by_keyword_hash)
      WHERE keyword_hash IN ({})
    """.format(", ".join(["%s"] * len(result)))
    args = list(iterkeys(hash_to_kw))
    if start_time:
      query += " AND timestamp >= FROM_UNIXTIME(%s)"
      args.append(mysql_utils.RDFDatetimeToTimestamp(start_time))
    cursor.execute(query, args)

    for kw_hash, cid in cursor.fetchall():
      result[hash_to_kw[kw_hash]].append(db_utils.IntToClientID(cid))
    return result

Response: Lists the clients associated with keywords.


Instruction:
Explain what the following Python 3 code does
def AddClientLabels(self, client_id, owner, labels, cursor=None):
    cid = db_utils.ClientIDToInt(client_id)
    labels = set(labels)
    args = [(cid, mysql_utils.Hash(owner), owner, label) for label in labels]
    args = list(collection.Flatten(args))

    query = """
          INSERT IGNORE INTO client_labels
              (client_id, owner_username_hash, owner_username, label)
          VALUES {}
          """.format(", ".join(["(%s, %s, %s, %s)"] * len(labels)))
    try:
      cursor.execute(query, args)
    except MySQLdb.IntegrityError as e:
      raise db.UnknownClientError(client_id, cause=e)

Response: Adds a list of user labels to a client.


Instruction:
Can you implement a function in Python 3 that
reads the user labels for a list of clients.

Response:
def MultiReadClientLabels(self, client_ids, cursor=None):
    """Reads the user labels for a list of clients."""

    int_ids = [db_utils.ClientIDToInt(cid) for cid in client_ids]
    query = ("SELECT client_id, owner_username, label "
             "FROM client_labels "
             "WHERE client_id IN ({})").format(", ".join(["%s"] *
                                                         len(client_ids)))

    ret = {client_id: [] for client_id in client_ids}
    cursor.execute(query, int_ids)
    for client_id, owner, label in cursor.fetchall():
      ret[db_utils.IntToClientID(client_id)].append(
          rdf_objects.ClientLabel(name=label, owner=owner))

    for r in itervalues(ret):
      r.sort(key=lambda label: (label.owner, label.name))
    return ret


Instruction:
Implement a function in Python 3 to
remove a list of user labels from a given client.

Response:
def RemoveClientLabels(self, client_id, owner, labels, cursor=None):
    """Removes a list of user labels from a given client."""

    query = ("DELETE FROM client_labels "
             "WHERE client_id = %s AND owner_username_hash = %s "
             "AND label IN ({})").format(", ".join(["%s"] * len(labels)))
    args = [db_utils.ClientIDToInt(client_id), mysql_utils.Hash(owner)] + labels
    cursor.execute(query, args)


Instruction:
Write a Python 3 function for
reading the user labels for a list of clients.

Response:
def ReadAllClientLabels(self, cursor=None):
    """Reads the user labels for a list of clients."""

    cursor.execute("SELECT DISTINCT owner_username, label FROM client_labels")

    result = []
    for owner, label in cursor.fetchall():
      result.append(rdf_objects.ClientLabel(name=label, owner=owner))

    result.sort(key=lambda label: (label.owner, label.name))
    return result


Instruction:
Write a Python 3 script to
write a new client crash record.

Response:
def WriteClientCrashInfo(self, client_id, crash_info, cursor=None):
    """Writes a new client crash record."""
    query = """
    SET @now = NOW(6);

    INSERT INTO client_crash_history (client_id, timestamp, crash_info)
         VALUES (%(client_id)s, @now, %(crash_info)s);

    UPDATE clients
       SET last_crash_timestamp = @now
     WHERE client_id = %(client_id)s
    """

    params = {
        "client_id": db_utils.ClientIDToInt(client_id),
        "crash_info": crash_info.SerializeToString(),
    }

    try:
      cursor.execute(query, params)
    except MySQLdb.IntegrityError as e:
      raise db.UnknownClientError(client_id, cause=e)


Instruction:
Write a Python 3 script to
read the latest client crash record for a single client.

Response:
def ReadClientCrashInfo(self, client_id, cursor=None):
    """Reads the latest client crash record for a single client."""
    cursor.execute(
        "SELECT UNIX_TIMESTAMP(timestamp), crash_info "
        "FROM clients, client_crash_history WHERE "
        "clients.client_id = client_crash_history.client_id AND "
        "clients.last_crash_timestamp = client_crash_history.timestamp AND "
        "clients.client_id = %s", [db_utils.ClientIDToInt(client_id)])
    row = cursor.fetchone()
    if not row:
      return None

    timestamp, crash_info = row
    res = rdf_client.ClientCrash.FromSerializedString(crash_info)
    res.timestamp = mysql_utils.TimestampToRDFDatetime(timestamp)
    return res


Instruction:
Write a Python 3 function for
reading the full crash history for a particular client.

Response:
def ReadClientCrashInfoHistory(self, client_id, cursor=None):
    """Reads the full crash history for a particular client."""
    cursor.execute(
        "SELECT UNIX_TIMESTAMP(timestamp), crash_info "
        "FROM client_crash_history WHERE "
        "client_crash_history.client_id = %s "
        "ORDER BY timestamp DESC", [db_utils.ClientIDToInt(client_id)])
    ret = []
    for timestamp, crash_info in cursor.fetchall():
      ci = rdf_client.ClientCrash.FromSerializedString(crash_info)
      ci.timestamp = mysql_utils.TimestampToRDFDatetime(timestamp)
      ret.append(ci)
    return ret


Instruction:
Can you create a Python 3 function that
writes a ClientStats instance to the database.

Response:
def WriteClientStats(self,
                       client_id,
                       stats,
                       cursor=None):
    """Stores a ClientStats instance."""

    try:
      cursor.execute(
          """
          INSERT INTO client_stats (client_id, payload, timestamp)
          VALUES (%s, %s, FROM_UNIXTIME(%s))
          ON DUPLICATE KEY UPDATE payload=VALUES(payload)
          """, [
              db_utils.ClientIDToInt(client_id),
              stats.SerializeToString(),
              mysql_utils.RDFDatetimeToTimestamp(rdfvalue.RDFDatetime.Now())
          ])
    except MySQLdb.IntegrityError as e:
      if e.args[0] == mysql_error_constants.NO_REFERENCED_ROW_2:
        raise db.UnknownClientError(client_id, cause=e)
      else:
        raise


Instruction:
Make a summary of the following Python 3 code
def ReadClientStats(self,
                      client_id,
                      min_timestamp,
                      max_timestamp,
                      cursor=None):
    """Reads ClientStats for a given client and time range."""

    cursor.execute(
        """
        SELECT payload FROM client_stats
        WHERE client_id = %s
          AND timestamp BETWEEN FROM_UNIXTIME(%s) AND FROM_UNIXTIME(%s)
        ORDER BY timestamp ASC
        """, [
            db_utils.ClientIDToInt(client_id),
            mysql_utils.RDFDatetimeToTimestamp(min_timestamp),
            mysql_utils.RDFDatetimeToTimestamp(max_timestamp)
        ])
    return [
        rdf_client_stats.ClientStats.FromSerializedString(stats_bytes)
        for stats_bytes, in cursor.fetchall()
    ]

Response: Reads ClientStats for a given client and time range.


Instruction:
Make a summary of the following Python 3 code
def DeleteOldClientStats(self, yield_after_count,
                           retention_time
                          ):
    """Deletes ClientStats older than a given timestamp."""

    yielded = False

    while True:
      deleted_count = self._DeleteClientStats(
          limit=yield_after_count, retention_time=retention_time)

      # Do not yield a trailing 0 after any non-zero count has been yielded.
      # A trailing zero occurs, when an exact multiple of `yield_after_count`
      # rows were in the table.
      if not yielded or deleted_count > 0:
        yield deleted_count
        yielded = True

      # Return, when no more rows can be deleted, indicated by a transaction
      # that does not reach the deletion limit.
      if deleted_count < yield_after_count:
        return

Response: Deletes ClientStats older than a given timestamp.


Instruction:
Make a summary of the following Python 3 code
def _DeleteClientStats(self,
                         limit,
                         retention_time,
                         cursor=None):
    """Deletes up to `limit` ClientStats older than `retention_time`."""
    cursor.execute(
        "DELETE FROM client_stats WHERE timestamp < FROM_UNIXTIME(%s) LIMIT %s",
        [mysql_utils.RDFDatetimeToTimestamp(retention_time), limit])
    return cursor.rowcount

Response: Deletes up to limit ClientStats older than retention_time.


Instruction:
Write a Python 3 function for
counting the client - activity metrics for a given statistic.

Response:
def _CountClientStatisticByLabel(self, statistic, day_buckets, cursor):
    """Returns client-activity metrics for a given statistic.

    Args:
      statistic: The name of the statistic, which should also be a column in the
        'clients' table.
      day_buckets: A set of n-day-active buckets.
      cursor: MySQL cursor for executing queries.
    """
    day_buckets = sorted(day_buckets)
    sum_clauses = []
    ping_cast_clauses = []
    timestamp_buckets = []
    now = rdfvalue.RDFDatetime.Now()

    for day_bucket in day_buckets:
      column_name = "days_active_{}".format(day_bucket)
      sum_clauses.append(
          "CAST(SUM({0}) AS UNSIGNED) AS {0}".format(column_name))
      ping_cast_clauses.append(
          "CAST(c.last_ping > FROM_UNIXTIME(%s) AS UNSIGNED) AS {}".format(
              column_name))
      timestamp_bucket = now - rdfvalue.Duration.FromDays(day_bucket)
      timestamp_buckets.append(
          mysql_utils.RDFDatetimeToTimestamp(timestamp_bucket))

    query = """
    SELECT j.{statistic}, j.label, {sum_clauses}
    FROM (
      SELECT c.{statistic} AS {statistic}, l.label AS label, {ping_cast_clauses}
      FROM clients c
      LEFT JOIN client_labels l USING(client_id)
      WHERE c.last_ping IS NOT NULL AND l.owner_username = 'GRR'
    ) AS j
    GROUP BY j.{statistic}, j.label
    """.format(
        statistic=statistic,
        sum_clauses=", ".join(sum_clauses),
        ping_cast_clauses=", ".join(ping_cast_clauses))

    cursor.execute(query, timestamp_buckets)

    counts = {}
    for response_row in cursor.fetchall():
      statistic_value, client_label = response_row[:2]
      for i, num_actives in enumerate(response_row[2:]):
        if num_actives <= 0:
          continue
        stats_key = (statistic_value, client_label, day_buckets[i])
        counts[stats_key] = num_actives
    return counts


Instruction:
Create a Python 3 function for
parsing a string path component into a PathComponent instance.

Response:
def ParsePathItem(item, opts=None):
  """Parses string path component to an `PathComponent` instance.

  Args:
    item: A path component string to be parsed.
    opts: A `PathOpts` object.

  Returns:
    `PathComponent` instance corresponding to given path fragment.

  Raises:
    ValueError: If the path item contains a recursive component fragment but
      cannot be parsed as such.
  """
  if item == os.path.curdir:
    return CurrentComponent()

  if item == os.path.pardir:
    return ParentComponent()

  recursion = PATH_RECURSION_REGEX.search(item)
  if recursion is None:
    return GlobComponent(item, opts)

  start, end = recursion.span()
  if not (start == 0 and end == len(item)):
    raise ValueError("malformed recursive component")

  if recursion.group("max_depth"):
    max_depth = int(recursion.group("max_depth"))
  else:
    max_depth = None

  return RecursiveComponent(max_depth=max_depth, opts=opts)


Instruction:
Write a Python 3 script to
parse a given path into a stream of PathComponent instances.

Response:
def ParsePath(path,
              opts = None):
  """Parses given path into a stream of `PathComponent` instances.

  Args:
    path: A path to be parsed.
    opts: An `PathOpts` object.

  Yields:
    `PathComponent` instances corresponding to the components of the given path.

  Raises:
    ValueError: If path contains more than one recursive component.
  """
  precondition.AssertType(path, Text)

  rcount = 0

  # Split the path at all forward slashes and if running under Windows, also
  # backward slashes. This allows ParsePath to handle native paths and also
  # normalized VFS paths like /HKEY_LOCAL_MACHINE/SAM.
  normalized_path = path.replace(os.path.sep, "/")

  for item in normalized_path.split("/"):
    component = ParsePathItem(item, opts=opts)
    if isinstance(component, RecursiveComponent):
      rcount += 1
      if rcount > 1:
        raise ValueError("path cannot have more than one recursive component")
    yield component


Instruction:
Implement a Python 3 function for
applying all expansion mechanisms to a given path.

Response:
def ExpandPath(path, opts=None):
  """Applies all expansion mechanisms to the given path.

  Args:
    path: A path to expand.
    opts: A `PathOpts` object.

  Yields:
    All paths possible to obtain from a given path by performing expansions.
  """
  precondition.AssertType(path, Text)

  for grouped_path in ExpandGroups(path):
    for globbed_path in ExpandGlobs(grouped_path, opts):
      yield globbed_path


Instruction:
Here you have a function in Python 3, explain what it does
def ExpandGroups(path):
  precondition.AssertType(path, Text)

  chunks = []
  offset = 0

  for match in PATH_GROUP_REGEX.finditer(path):
    chunks.append([path[offset:match.start()]])
    chunks.append(match.group("alts").split(","))
    offset = match.end()

  chunks.append([path[offset:]])

  for prod in itertools.product(*chunks):
    yield "".join(prod)

Response: Performs group expansion on a given path.


Instruction:
Write a Python 3 script to
perform glob expansion on a given path.

Response:
def ExpandGlobs(path, opts = None):
  """Performs glob expansion on a given path.

  Path can contain regular glob elements (such as `**`, `*`, `?`, `[a-z]`). For
  example, having files `foo`, `bar`, `baz` glob expansion of `ba?` will yield
  `bar` and `baz`.

  Args:
    path: A path to expand.
    opts: A `PathOpts` object.

  Returns:
    Generator over all possible glob expansions of a given path.

  Raises:
    ValueError: If given path is empty or relative.
  """
  precondition.AssertType(path, Text)
  if not path:
    raise ValueError("Path is empty")

  if not _IsAbsolutePath(path, opts):
    raise ValueError("Path '%s' is not absolute" % path)

  if opts is not None and opts.pathtype == rdf_paths.PathSpec.PathType.REGISTRY:
    # Handle HKLM\Foo and /HKLM/Foo identically.
    root_dir, tail = path.replace("\\", "/").lstrip("/").split("/", 1)
    components = list(ParsePath(tail, opts=opts))
  else:
    drive, tail = os.path.splitdrive(path)
    root_dir = os.path.join(drive, os.path.sep).upper()
    components = list(ParsePath(tail[1:], opts=opts))

  return _ExpandComponents(root_dir, components)


Instruction:
How would you explain what the following Python 3 function does
def _ListDir(dirpath, pathtype):
  pathspec = rdf_paths.PathSpec(path=dirpath, pathtype=pathtype)
  childpaths = []
  try:
    file_obj = vfs.VFSOpen(pathspec)
    for path in file_obj.ListNames():
      # For Windows registry, ignore the empty string which corresponds to the
      # default value in the current key. Otherwise, globbing a key will yield
      # the key itself, because joining the name of the default value u"" with
      # a key name yields the key name again.
      if pathtype != rdf_paths.PathSpec.PathType.REGISTRY or path:
        childpaths.append(path)
  except IOError:
    pass

  return childpaths

Response: Returns a list of potential children of a given directory.


Instruction:
Here you have a function in Python 3, explain what it does
def ListNetworkConnectionsFromClient(args):
  for proc in psutil.process_iter():
    try:
      connections = proc.connections()
    except (psutil.NoSuchProcess, psutil.AccessDenied):
      continue

    for conn in connections:
      if args.listening_only and conn.status != "LISTEN":
        continue

      res = rdf_client_network.NetworkConnection()
      res.pid = proc.pid
      res.process_name = proc.name()
      res.family = conn.family
      res.type = conn.type
      try:
        if conn.status:
          res.state = conn.status
      except ValueError:
        logging.warning("Encountered unknown connection status (%s).",
                        conn.status)

      res.local_address.ip, res.local_address.port = conn.laddr
      if conn.raddr:
        res.remote_address.ip, res.remote_address.port = conn.raddr

      yield res

Response: Gather open network connection stats.


Instruction:
How would you explain what the following Python 3 function does
def IterIfaddrs(ifaddrs):
  precondition.AssertOptionalType(ifaddrs, ctypes.POINTER(Ifaddrs))

  while ifaddrs:
    yield ifaddrs.contents
    ifaddrs = ifaddrs.contents.ifa_next

Response: Iterates over contents of the intrusive linked list of ifaddrs.


Instruction:
Can you create a Python 3 function that
parses the contents of the intrusive linked list of ifaddrs.

Response:
def ParseIfaddrs(ifaddrs):
  """Parses contents of the intrusive linked list of `ifaddrs`.

  Args:
    ifaddrs: A pointer to the first node of `ifaddrs` linked list. Can be NULL.

  Returns:
    An iterator over instances of `rdf_client_network.Interface`.
  """
  precondition.AssertOptionalType(ifaddrs, ctypes.POINTER(Ifaddrs))

  ifaces = {}

  for ifaddr in IterIfaddrs(ifaddrs):
    ifname = ctypes.string_at(ifaddr.ifa_name).decode("utf-8")
    iface = ifaces.setdefault(ifname, rdf_client_network.Interface())
    iface.ifname = ifname

    if not ifaddr.ifa_addr:
      continue

    sockaddr = ctypes.cast(ifaddr.ifa_addr, ctypes.POINTER(Sockaddr))
    iffamily = sockaddr.contents.sa_family
    if iffamily == AF_INET:
      sockaddrin = ctypes.cast(ifaddr.ifa_addr, ctypes.POINTER(Sockaddrin))

      address = rdf_client_network.NetworkAddress()
      address.address_type = rdf_client_network.NetworkAddress.Family.INET
      address.packed_bytes = struct.pack("=L", sockaddrin.contents.sin_addr)
      iface.addresses.append(address)
    elif iffamily == AF_INET6:
      sockaddrin = ctypes.cast(ifaddr.ifa_addr, ctypes.POINTER(Sockaddrin6))

      address = rdf_client_network.NetworkAddress()
      address.address_type = rdf_client_network.NetworkAddress.Family.INET6
      address.packed_bytes = bytes(list(sockaddrin.contents.sin6_addr))
      iface.addresses.append(address)
    elif iffamily == AF_LINK:
      sockaddrdl = ctypes.cast(ifaddr.ifa_addr, ctypes.POINTER(Sockaddrdl))

      nlen = sockaddrdl.contents.sdl_nlen
      alen = sockaddrdl.contents.sdl_alen
      iface.mac_address = bytes(sockaddrdl.contents.sdl_data[nlen:nlen + alen])
    else:
      raise ValueError("Unexpected socket address family: %s" % iffamily)

  return itervalues(ifaces)


Instruction:
Implement a function in Python 3 to
enumerate all MAC addresses.

Response:
def EnumerateInterfacesFromClient(args):
  """Enumerate all MAC addresses."""
  del args  # Unused

  libc = ctypes.cdll.LoadLibrary(ctypes.util.find_library("c"))
  ifa = Ifaddrs()
  p_ifa = ctypes.pointer(ifa)
  libc.getifaddrs(ctypes.pointer(p_ifa))

  for iface in ParseIfaddrs(p_ifa):
    yield iface

  libc.freeifaddrs(p_ifa)


Instruction:
Make a summary of the following Python 3 code
def EnumerateFilesystemsFromClient(args):
  del args  # Unused.
  for fs_struct in client_utils_osx.GetFileSystems():
    yield rdf_client_fs.Filesystem(
        device=fs_struct.f_mntfromname,
        mount_point=fs_struct.f_mntonname,
        type=fs_struct.f_fstypename)

  drive_re = re.compile("r?disk[0-9].*")
  for drive in os.listdir("/dev"):
    if not drive_re.match(drive):
      continue

    path = os.path.join("/dev", drive)
    try:
      img_inf = pytsk3.Img_Info(path)
      # This is a volume or a partition - we send back a TSK device.
      yield rdf_client_fs.Filesystem(device=path)

      vol_inf = pytsk3.Volume_Info(img_inf)

      for volume in vol_inf:
        if volume.flags == pytsk3.TSK_VS_PART_FLAG_ALLOC:
          offset = volume.start * vol_inf.info.block_size
          yield rdf_client_fs.Filesystem(
              device="{path}:{offset}".format(path=path, offset=offset),
              type="partition")

    except (IOError, RuntimeError):
      continue

Response: Enumerate all local filesystems mounted on this system.


Instruction:
How would you code a function in Python 3 to
create the Service protobuf.

Response:
def CreateServiceProto(job):
  """Create the Service protobuf.

  Args:
    job: Launchdjobdict from servicemanagement framework.

  Returns:
    sysinfo_pb2.OSXServiceInformation proto
  """
  service = rdf_client.OSXServiceInformation(
      label=job.get("Label"),
      program=job.get("Program"),
      sessiontype=job.get("LimitLoadToSessionType"),
      lastexitstatus=int(job["LastExitStatus"]),
      timeout=int(job["TimeOut"]),
      ondemand=bool(job["OnDemand"]))

  for arg in job.get("ProgramArguments", "", stringify=False):
    # Returns CFArray of CFStrings
    service.args.Append(str(arg))

  mach_dict = job.get("MachServices", {}, stringify=False)
  for key, value in iteritems(mach_dict):
    service.machservice.Append("%s:%s" % (key, value))

  job_mach_dict = job.get("PerJobMachServices", {}, stringify=False)
  for key, value in iteritems(job_mach_dict):
    service.perjobmachservice.Append("%s:%s" % (key, value))

  if "PID" in job:
    service.pid = job["PID"].value

  return service


Instruction:
Implement a Python 3 function for
enumerating running launchd jobs.

Response:
def OSXEnumerateRunningServicesFromClient(args):
  """Get running launchd jobs.

  Args:
    args: Unused.

  Yields:
    `rdf_client.OSXServiceInformation` instances.

  Raises:
      UnsupportedOSVersionError: for OS X earlier than 10.6.
  """
  del args  # Unused.
  osx_version = client_utils_osx.OSXVersion()
  version_array = osx_version.VersionAsMajorMinor()

  if version_array[:2] < [10, 6]:
    raise UnsupportedOSVersionError(
        "ServiceManagement API unsupported on < 10.6. This client is %s" %
        osx_version.VersionString())

  launchd_list = GetRunningLaunchDaemons()

  parser = osx_launchd.OSXLaunchdJobDict(launchd_list)
  for job in parser.Parse():
    response = CreateServiceProto(job)
    yield response


Instruction:
Can you generate the documentation for the following Python 3 function
def Run(self, unused_arg):
    logging.debug("Disabling service")

    msg = "Service disabled."
    if hasattr(sys, "frozen"):
      grr_binary = os.path.abspath(sys.executable)
    elif __file__:
      grr_binary = os.path.abspath(__file__)

    try:
      os.remove(grr_binary)
    except OSError:
      msg = "Could not remove binary."

    try:
      os.remove(config.CONFIG["Client.plist_path"])
    except OSError:
      if "Could not" in msg:
        msg += " Could not remove plist file."
      else:
        msg = "Could not remove plist file."

    # Get the directory we are running in from pyinstaller. This is either the
    # GRR directory which we should delete (onedir mode) or a generated temp
    # directory which we can delete without problems in onefile mode.
    directory = getattr(sys, "_MEIPASS", None)
    if directory:
      shutil.rmtree(directory, ignore_errors=True)

    self.SendReply(rdf_protodict.DataBlob(string=msg))

Response: This kills us with no cleanups.


Instruction:
Write a Python 3 script for
starting remote debugging using pydevd.

Response:
def _start_remote_debugging(port):
  """Sets up remote debugging using pydevd, connecting to localhost:`port`."""
  try:
    print("Connecting to remote debugger on localhost:{}.".format(port))
    import pydevd  # pylint: disable=g-import-not-at-top
    pydevd.settrace(
        "localhost",
        port=port,
        stdoutToServer=True,
        stderrToServer=True,
        suspend=flags.FLAGS.break_on_start)
  except ImportError:
    print(
        "pydevd is required for remote debugging. Please follow the PyCharm"
        "manual or run `pip install pydevd-pycharm` to install.",
        file=sys.stderr)


Instruction:
Here you have a function in Python 3, explain what it does
def ExtractConfig(self):
    logging.info("Extracting config file from .pkg.")
    pkg_path = os.environ.get("PACKAGE_PATH", None)
    if pkg_path is None:
      logging.error("Could not locate package, giving up.")
      return

    zf = zipfile.ZipFile(pkg_path, mode="r")
    fd = zf.open("config.yaml")
    install_dir = os.path.dirname(config.CONFIG.parser.filename)

    # We write this config to disk so that Intialize can find the build.yaml
    # referenced inside the config as a relative path. This config isn't used
    # after install time.
    installer_config = os.path.join(install_dir, "installer_config.yaml")
    with open(installer_config, "wb") as f:
      f.write(fd.read())

    packaged_config = config.CONFIG.MakeNewConfig()
    packaged_config.Initialize(
        filename=installer_config, parser=config_lib.YamlParser)

    new_config = config.CONFIG.MakeNewConfig()
    new_config.SetWriteBack(config.CONFIG["Config.writeback"])

    for info in config.CONFIG.type_infos:
      try:
        new_value = packaged_config.GetRaw(info.name, None)
      except type_info.TypeValueError:
        continue

      try:
        old_value = config.CONFIG.GetRaw(info.name, None)

        if not new_value or new_value == old_value:
          continue
      except type_info.TypeValueError:
        pass

      new_config.SetRaw(info.name, new_value)

    new_config.Write()
    logging.info("Config file extracted successfully.")

    logging.info("Extracting additional files.")

    for zinfo in zf.filelist:
      basename = os.path.basename(zinfo.filename)
      if basename != "config.yaml":
        with open(os.path.join(install_dir, basename), "wb") as f:
          f.write(zf.open(zinfo.filename).read())

Response: This installer extracts a config file from the. pkg file.


Instruction:
Can you tell what is the following Python 3 function doing
def _LoadFlows(self, client_id, min_create_time, token):
    if data_store.RelationalDBEnabled():
      if isinstance(client_id, rdfvalue.RDFURN):
        client_id = client_id.Basename()

      flow_list = data_store.REL_DB.ReadAllFlowObjects(
          client_id=client_id,
          min_create_time=min_create_time,
          include_child_flows=False)
      for flow_obj in flow_list:
        yield flow_obj
    else:
      now = rdfvalue.RDFDatetime.Now()
      client_id_urn = rdf_client.ClientURN(client_id)
      flows_dir = aff4.FACTORY.Open(client_id_urn.Add("flows"), token=token)
      # Save DB roundtrips by checking both conditions at once.
      flow_list = flows_dir.ListChildren(
          age=(min_create_time.AsMicrosecondsSinceEpoch(),
               now.AsMicrosecondsSinceEpoch()))
      for flow_obj in aff4.FACTORY.MultiOpen(flow_list, token=token):
        yield rdf_flow_objects.Flow(
            args=flow_obj.args,
            flow_class_name=flow_obj.runner_args.flow_name,
            flow_id=flow_obj.urn.Basename(),
            create_time=flow_obj.context.create_time,
            creator=flow_obj.creator,
        )

Response: Load all flows for a given client_id and time range.


Instruction:
Here you have a function in Python 3, explain what it does
def EnforceLimits(self, client_id, user, flow_name, flow_args, token=None):
    if not self.dup_interval and not self.daily_req_limit:
      return

    now = rdfvalue.RDFDatetime.Now()
    yesterday = now - rdfvalue.Duration("1d")
    dup_boundary = now - self.dup_interval
    min_create_time = min(yesterday, dup_boundary)

    flow_count = 0
    flows = self._LoadFlows(client_id, min_create_time, token=token)

    if flow_args is None:
      flow_args = flow.EmptyFlowArgs()

    for flow_obj in flows:
      if (flow_obj.create_time > dup_boundary and
          flow_obj.flow_class_name == flow_name and flow_obj.args == flow_args):
        raise DuplicateFlowError(
            "Identical %s already run on %s at %s" %
            (flow_name, client_id, flow_obj.create_time),
            flow_id=flow_obj.flow_id)

      # Filter for flows started by user within the 1 day window.
      if flow_obj.creator == user and flow_obj.create_time > yesterday:
        flow_count += 1

    # If limit is set, enforce it.
    if self.daily_req_limit and flow_count >= self.daily_req_limit:
      raise DailyFlowRequestLimitExceededError(
          "%s flows run since %s, limit: %s" %
          (flow_count, yesterday, self.daily_req_limit))

Response: Enforces DailyFlowRequestLimit and FlowDuplicateInterval.


Instruction:
Can you write a function in Python 3 where it
builds a StatEntry object from a given path.

Response:
def StatEntryFromPath(path, pathspec, ext_attrs=True):
  """Builds a stat entry object from a given path.

  Args:
    path: A path (string value) to stat.
    pathspec: A `PathSpec` corresponding to the `path`.
    ext_attrs: Whether to include extended file attributes in the result.

  Returns:
    `StatEntry` object.
  """
  try:
    stat = filesystem.Stat.FromPath(path)
  except (IOError, OSError) as error:
    logging.error("Failed to obtain stat for '%s': %s", pathspec, error)
    return rdf_client_fs.StatEntry(pathspec=pathspec)

  return StatEntryFromStat(stat, pathspec, ext_attrs=ext_attrs)


Instruction:
How would you explain what the following Python 3 function does
def StatEntryFromStat(stat,
                      pathspec,
                      ext_attrs = True):
  """Build a stat entry object from a given stat object.

  Args:
    stat: A `Stat` object.
    pathspec: A `PathSpec` from which `stat` was obtained.
    ext_attrs: Whether to include extended file attributes in the result.

  Returns:
    `StatEntry` object.
  """
  result = rdf_client_fs.StatEntry(pathspec=pathspec)

  for attr in _STAT_ATTRS:
    value = getattr(stat.GetRaw(), attr, None)
    if value is None:
      continue

    # TODO(hanuszczak): Why are we doing this?
    value = int(value)
    if value < 0:
      value &= 0xFFFFFFFF

    setattr(result, attr, value)

  result.st_flags_linux = stat.GetLinuxFlags()
  result.st_flags_osx = stat.GetOsxFlags()
  if ext_attrs:
    # TODO(hanuszczak): Can we somehow incorporate extended attribute getter to
    # the `Stat` class? That would make the code a lot prettier but would force
    # `utils` to depend on `xattrs`.
    result.ext_attrs = list(GetExtAttrs(stat.GetPath()))

  return result

Response: Builds a stat entry object from a given stat object.


Instruction:
Implement a Python 3 function for
returning a os. stat_result object with most information from a StatEntry.

Response:
def StatResultFromStatEntry(
    stat_entry):
  """Returns a `os.stat_result` with most information from `StatEntry`.

  This is a lossy conversion, only the 10 first stat_result fields are
  populated, because the os.stat_result constructor is inflexible.

  Args:
    stat_entry: An instance of rdf_client_fs.StatEntry.

  Returns:
    An instance of `os.stat_result` with basic fields populated.
  """
  values = []
  for attr in _STAT_ATTRS[:10]:
    values.append(stat_entry.Get(attr))
  return os.stat_result(values)


Instruction:
Write a Python 3 script for
initializing Api ( Client Hunt or CronJob ) Approval from an AFF4 object.

Response:
def _InitApiApprovalFromAff4Object(api_approval, approval_obj):
  """Initializes Api(Client|Hunt|CronJob)Approval from an AFF4 object."""

  api_approval.id = approval_obj.urn.Basename()
  api_approval.reason = approval_obj.Get(approval_obj.Schema.REASON)
  api_approval.requestor = approval_obj.Get(approval_obj.Schema.REQUESTOR)

  # We should check the approval validity from the standpoint of the user
  # who had requested it.
  test_token = access_control.ACLToken(
      username=approval_obj.Get(approval_obj.Schema.REQUESTOR))
  try:
    approval_obj.CheckAccess(test_token)
    api_approval.is_valid = True
  except access_control.UnauthorizedAccess as e:
    api_approval.is_valid = False
    api_approval.is_valid_message = utils.SmartStr(e)

  notified_users = approval_obj.Get(approval_obj.Schema.NOTIFIED_USERS)
  if notified_users:
    api_approval.notified_users = sorted(
        u.strip() for u in notified_users.split(","))

  api_approval.email_message_id = approval_obj.Get(
      approval_obj.Schema.EMAIL_MSG_ID)

  email_cc = approval_obj.Get(approval_obj.Schema.EMAIL_CC)
  email_cc_addresses = sorted(s.strip() for s in email_cc.split(","))
  api_approval.email_cc_addresses = (
      set(email_cc_addresses) - set(api_approval.notified_users))

  api_approval.approvers = sorted(approval_obj.GetNonExpiredApprovers())

  return api_approval


Instruction:
How would you code a function in Python 3 to
initialize Api ( Client Hunt | CronJob ) Approval from the database object.

Response:
def _InitApiApprovalFromDatabaseObject(api_approval, db_obj):
  """Initializes Api(Client|Hunt|CronJob)Approval from the database object."""

  api_approval.id = db_obj.approval_id
  api_approval.requestor = db_obj.requestor_username
  api_approval.reason = db_obj.reason

  api_approval.notified_users = sorted(db_obj.notified_users)
  api_approval.email_cc_addresses = sorted(db_obj.email_cc_addresses)
  api_approval.email_message_id = db_obj.email_message_id

  api_approval.approvers = sorted([g.grantor_username for g in db_obj.grants])

  try:
    approval_checks.CheckApprovalRequest(db_obj)
    api_approval.is_valid = True
  except access_control.UnauthorizedAccess as e:
    api_approval.is_valid_message = utils.SmartStr(e)
    api_approval.is_valid = False

  return api_approval