Instruction:
How would you explain what the following Python 3 function does
def Validate(self):
    if not self.check_id:
      raise DefinitionError("Check has missing check_id value")
    cls_name = self.check_id
    if not self.method:
      raise DefinitionError("Check %s has no methods" % cls_name)
    ValidateMultiple(self.method,
                     "Check %s has invalid method definitions" % cls_name)

Response: Validate the check method.


Instruction:
Can you tell what is the following Python 3 function doing
def Detect(self, baseline, host_data):
    result = CheckResult()
    for detector in self.detectors:
      finding = detector(baseline, host_data)
      if finding:
        result.ExtendAnomalies([finding])
    if result:
      return result

Response: Runs through detectors and returns anomalies if any detector triggers the issue.


Instruction:
Explain what the following Python 3 code does
def Issue(self, state, results):
    result = CheckResult()
    # If there are CheckResults we're aggregating methods or probes.
    # Merge all current results into one CheckResult.
    # Otherwise, the results are raw host data.
    # Generate a new CheckResult and add the specific findings.
    if results and all(isinstance(r, CheckResult) for r in results):
      result.ExtendAnomalies(results)
    else:
      result.anomaly = [
          rdf_anomaly.Anomaly(
              type=anomaly_pb2.Anomaly.AnomalyType.Name(
                  anomaly_pb2.Anomaly.ANALYSIS_ANOMALY),
              symptom=self.hint.Problem(state),
              finding=self.hint.Render(results),
              explanation=self.hint.Fix())
      ]
    return result

Response: Issues an anomalous findings.


Instruction:
How would you code a function in Python 3 to
register a check in the registry.

Response:
def RegisterCheck(cls, check, source="unknown", overwrite_if_exists=False):
    """Adds a check to the registry, refresh the trigger to check map."""
    if not overwrite_if_exists and check.check_id in cls.checks:
      raise DefinitionError(
          "Check named %s already exists and "
          "overwrite_if_exists is set to False." % check.check_id)
    check.loaded_from = source
    cls.checks[check.check_id] = check
    cls.triggers.Update(check.triggers, check)


Instruction:
Given the following Python 3 function, write the documentation
def _AsList(arg):
    if (isinstance(arg, string_types) or
        not isinstance(arg, collections.Iterable)):
      return [arg]
    else:
      return list(arg)

Response: Encapsulates an argument in a list if it s not already iterable.


Instruction:
Can you generate the documentation for the following Python 3 function
def Conditions(cls, artifact=None, os_name=None, cpe=None, labels=None):
    artifact = cls._AsList(artifact)
    os_name = cls._AsList(os_name)
    cpe = cls._AsList(cpe)
    labels = cls._AsList(labels)
    for condition in itertools.product(artifact, os_name, cpe, labels):
      yield condition

Response: Generates a series of condition tuples.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def FindChecks(cls,
                 artifact=None,
                 os_name=None,
                 cpe=None,
                 labels=None,
                 restrict_checks=None):
    """Takes targeting info, identifies relevant checks.

    FindChecks will return results when a host has the conditions necessary for
    a check to occur. Conditions with partial results are not returned. For
    example, FindChecks will not return checks that if a check targets
    os_name=["Linux"], labels=["foo"] and a host only has the os_name=["Linux"]
    attribute.

    Args:
      artifact: 0+ artifact names.
      os_name: 0+ OS names.
      cpe: 0+ CPE identifiers.
      labels: 0+ GRR labels.
      restrict_checks: A list of check ids to restrict check processing to.

    Returns:
      the check_ids that apply.
    """
    check_ids = set()
    conditions = list(cls.Conditions(artifact, os_name, cpe, labels))
    for chk_id, chk in iteritems(cls.checks):
      if restrict_checks and chk_id not in restrict_checks:
        continue
      for condition in conditions:
        if chk.triggers.Match(*condition):
          check_ids.add(chk_id)
          break  # No need to keep checking other conditions.
    return check_ids

Response: Returns a list of check ids that apply to the given set of conditions.


Instruction:
Can you tell what is the following Python 3 function doing
def SelectArtifacts(cls,
                      os_name=None,
                      cpe=None,
                      labels=None,
                      restrict_checks=None):
    """Takes targeting info, identifies artifacts to fetch.

    Args:
      os_name: 0+ OS names.
      cpe: 0+ CPE identifiers.
      labels: 0+ GRR labels.
      restrict_checks: A list of check ids whose artifacts should be fetched.

    Returns:
      the artifacts that should be collected.
    """
    results = set()
    for condition in cls.Conditions(None, os_name, cpe, labels):
      trigger = condition[1:]
      for chk in itervalues(cls.checks):
        if restrict_checks and chk.check_id not in restrict_checks:
          continue
        results.update(chk.triggers.Artifacts(*trigger))
    return results

Response: Takes targeting info identifies artifacts to fetch.


Instruction:
Write a Python 3 function for
processing the host data and returns a list of CheckResult messages.

Response:
def Process(cls,
              host_data,
              os_name=None,
              cpe=None,
              labels=None,
              exclude_checks=None,
              restrict_checks=None):
    """Runs checks over all host data.

    Args:
      host_data: The data collected from a host, mapped to artifact name.
      os_name: 0+ OS names.
      cpe: 0+ CPE identifiers.
      labels: 0+ GRR labels.
      exclude_checks: A list of check ids not to run. A check id in this list
                      will not get run even if included in restrict_checks.
      restrict_checks: A list of check ids that may be run, if appropriate.

    Yields:
      A CheckResult message for each check that was performed.
    """
    # All the conditions that apply to this host.
    artifacts = list(iterkeys(host_data))
    check_ids = cls.FindChecks(artifacts, os_name, cpe, labels)
    conditions = list(cls.Conditions(artifacts, os_name, cpe, labels))
    for check_id in check_ids:
      # skip if check in list of excluded checks
      if exclude_checks and check_id in exclude_checks:
        continue
      if restrict_checks and check_id not in restrict_checks:
        continue
      try:
        chk = cls.checks[check_id]
        yield chk.Parse(conditions, host_data)
      except ProcessingError as e:
        logging.warning("Check ID %s raised: %s", check_id, e)


Instruction:
Write a Python 3 script to
return a pseudo - random 32 - bit unsigned integer.

Response:
def UInt32():
  """Returns a pseudo-random 32-bit unsigned integer."""
  with _mutex:
    try:
      return _random_buffer.pop()
    except IndexError:
      data = os.urandom(struct.calcsize("=L") * _random_buffer_size)
      _random_buffer.extend(
          struct.unpack("=" + "L" * _random_buffer_size, data))
      return _random_buffer.pop()


Instruction:
Write a Python 3 script for
setting up some vars for the directories we use.

Response:
def SetupVars(self):
    """Set up some vars for the directories we use."""
    # Python paths chosen to match appveyor:
    # http://www.appveyor.com/docs/installed-software#python

    self.python_dir_64 = args.python64_dir
    self.python_dir_32 = args.python32_dir

    self.virtualenv_bin64 = os.path.join(self.python_dir_64,
                                         r"Scripts\virtualenv.exe")
    self.virtualenv_bin32 = os.path.join(self.python_dir_32,
                                         r"Scripts\virtualenv.exe")

    if args.virtualenv_64bit:
      self.virtualenv64 = args.virtualenv_64bit
    else:
      self.virtualenv64 = os.path.join(args.build_dir, "python_64")
    if args.virtualenv_32bit:
      self.virtualenv32 = args.virtualenv_32bit
    else:
      self.virtualenv32 = os.path.join(args.build_dir, "python_32")

    self.grr_client_build64 = os.path.join(self.virtualenv64,
                                           r"Scripts\grr_client_build.exe")
    self.grr_client_build32 = os.path.join(self.virtualenv32,
                                           r"Scripts\grr_client_build.exe")
    self.pip64 = os.path.join(self.virtualenv64, r"Scripts\pip.exe")
    self.pip32 = os.path.join(self.virtualenv32, r"Scripts\pip.exe")

    self.virtualenv_python64 = os.path.join(self.virtualenv64,
                                            r"Scripts\python.exe")
    self.virtualenv_python32 = os.path.join(self.virtualenv32,
                                            r"Scripts\python.exe")

    self.git = r"C:\Program Files\Git\bin\git.exe"

    self.install_path = r"C:\Windows\System32\GRR"
    self.service_name = "GRR Monitor"

    self.expect_service_running = args.expect_service_running


Instruction:
Here you have a function in Python 3, explain what it does
def Clean(self):
    # os.unlink doesn't work effectively, use the shell to delete.
    if os.path.exists(args.build_dir):
      subprocess.call("rd /s /q %s" % args.build_dir, shell=True)
    if os.path.exists(args.output_dir):
      subprocess.call("rd /s /q %s" % args.output_dir, shell=True)

    os.makedirs(args.build_dir)
    os.makedirs(args.output_dir)

    # Create virtualenvs.
    if not args.virtualenv_64bit:
      subprocess.check_call([self.virtualenv_bin64, self.virtualenv64])
    if args.build_32 and not args.virtualenv_32bit:
      subprocess.check_call([self.virtualenv_bin32, self.virtualenv32])

    # Currently this should do nothing as we will already have a modern pip
    # installed, but we leave this here so if we get broken by pip again it's
    # just a simple case of searching for pip>=8.1.1 and adding an upper limit
    # cap in all those places.

    cmd = ["-m", "pip", "install"]
    if args.wheel_dir:
      cmd += ["--no-index", r"--find-links=file:///%s" % args.wheel_dir]

    subprocess.check_call([self.virtualenv_python64] + cmd +
                          ["--upgrade", "pip>=8.1.1"])
    if args.build_32:
      subprocess.check_call([self.virtualenv_python32] + cmd +
                            ["--upgrade", "pip>=8.1.1"])
    os.environ["PROTOC"] = args.protoc

Response: Clean the build environment.


Instruction:
How would you explain what the following Python 3 function does
def BuildTemplates(self):
    if args.config:
      build_args = [
          "--verbose", "--config", args.config, "build", "--output",
          args.output_dir
      ]
    else:
      build_args = ["--verbose", "build", "--output", args.output_dir]
    subprocess.check_call([self.grr_client_build64] + build_args)
    if args.build_32:
      subprocess.check_call([self.grr_client_build32] + build_args)

Response: Builds the client templates.


Instruction:
Explain what the following Python 3 code does
def _CleanupInstall(self):
    if os.path.exists(self.install_path):
      shutil.rmtree(self.install_path)
      if os.path.exists(self.install_path):
        raise RuntimeError("Install path still exists: %s" % self.install_path)

    # Deliberately don't check return code, since service may not be installed.
    subprocess.call(["sc", "stop", self.service_name])

Response: Cleanup from any previous installer enough for _CheckInstallSuccess.


Instruction:
Here you have a function in Python 3, explain what it does
def _CheckInstallSuccess(self):
    if not os.path.exists(self.install_path):
      raise RuntimeError("Install failed, no files at: %s" % self.install_path)

    try:
      output = subprocess.check_output(["sc", "query", self.service_name])
      service_running = "RUNNING" in output
    except subprocess.CalledProcessError as e:
      if e.returncode == 1060:
        # 1060 means: The specified service does not exist as an installed
        # service.
        service_running = False
      else:
        raise

    if self.expect_service_running:
      if not service_running:
        raise RuntimeError(
            "GRR service not running after install, sc query output: %s" %
            output)
    else:
      if service_running:
        raise RuntimeError(
            "GRR service running after install with expect_service_running == "
            "False, sc query output: %s" % output)

Response: Checks if the installer installed correctly.


Instruction:
Can you tell what is the following Python 3 function doing
def _InstallInstallers(self):
    # 32 bit binary will refuse to install on a 64bit system so we only install
    # the 64 bit version
    installer_amd64 = glob.glob(
        os.path.join(args.output_dir, "dbg_*_amd64.exe")).pop()
    self._CleanupInstall()
    # The exit code is always 0, test to see if install was actually successful.
    subprocess.check_call([installer_amd64])
    self._CheckInstallSuccess()

Response: Install the installer built by RepackTemplates.


Instruction:
Create a Python 3 function to
ensure that given value has the expected type.

Response:
def AssertType(value, expected_type):
  """Ensures that given value has certain type.

  Args:
    value: A value to assert the type for.
    expected_type: An expected type for the given value.

  Raises:
    TypeError: If given value does not have the expected type.
  """
  if not isinstance(value, expected_type):
    message = "Expected type `%r`, but got value `%r` of type `%s`"
    message %= (expected_type, value, type(value))
    raise TypeError(message)


Instruction:
Can you tell what is the following Python 3 function doing
def AssertIterableType(iterable, expected_item_type):
  # We do not consider iterators to be iterables even though Python does. An
  # "iterable" should be a type that can be iterated (that is: an iterator can
  # be constructed for them). Iterators should not be considered to be iterable
  # because it makes no sense to construct an iterator for iterator. The most
  # important practical implication is that act of iterating an iterator drains
  # it whereas act of iterating the iterable does not.
  if isinstance(iterable, collections.Iterator):
    message = "Expected iterable container but got iterator `%s` instead"
    message %= iterable
    raise TypeError(message)

  AssertType(iterable, collections.Iterable)
  for item in iterable:
    AssertType(item, expected_item_type)

Response: Ensures that given iterable container has certain type.


Instruction:
Here you have a function in Python 3, explain what it does
def AssertDictType(dct, expected_key_type, expected_value_type):
  AssertType(dct, dict)
  for key, value in iteritems(dct):
    AssertType(key, expected_key_type)
    AssertType(value, expected_value_type)

Response: Ensures that given dictionary is actually a dictionary of specified type.


Instruction:
Write a Python 3 script to
run the single hook specified by resolving all its prerequisites.

Response:
def _RunSingleHook(self, hook_cls, executed_set, required=None):
    """Run the single hook specified by resolving all its prerequisites."""
    # If we already ran do nothing.
    if hook_cls in executed_set:
      return

    # Ensure all the pre execution hooks are run.
    for pre_hook in hook_cls.pre:
      self._RunSingleHook(pre_hook, executed_set, required=hook_cls.__name__)

    # Now run this hook.
    cls_instance = hook_cls()
    if required:
      logging.debug("Initializing %s, required by %s", hook_cls.__name__,
                    required)
    else:
      logging.debug("Initializing %s", hook_cls.__name__)

    # Always call the Run hook.
    cls_instance.Run()
    executed_set.add(hook_cls)

    # Only call the RunOnce() hook if not already called.
    if hook_cls not in self.already_run_once:
      cls_instance.RunOnce()
      self.already_run_once.add(hook_cls)


Instruction:
Implement a function in Python 3 to
initialize the object from a database hunt object.

Response:
def InitFromHuntObject(self,
                         hunt_obj,
                         hunt_counters=None,
                         with_full_summary=False):
    """Initialize API hunt object from a database hunt object.

    Args:
      hunt_obj: rdf_hunt_objects.Hunt to read the data from.
      hunt_counters: Optional db.HuntCounters object with counters information.
      with_full_summary: if True, hunt_runner_args, completion counts and a few
        other fields will be filled in. The way to think about it is that with
        with_full_summary==True ApiHunt will have the data to render "Hunt
        Overview" page and with with_full_summary==False it will have enough
        data to be rendered as a hunts list row.

    Returns:
      Self.
    """

    self.urn = rdfvalue.RDFURN("hunts").Add(str(hunt_obj.hunt_id))
    self.hunt_id = hunt_obj.hunt_id
    if (hunt_obj.args.hunt_type ==
        rdf_hunt_objects.HuntArguments.HuntType.STANDARD):
      self.name = "GenericHunt"
    else:
      self.name = "VariableGenericHunt"
    self.state = str(hunt_obj.hunt_state)
    self.crash_limit = hunt_obj.crash_limit
    self.client_limit = hunt_obj.client_limit
    self.client_rate = hunt_obj.client_rate
    self.created = hunt_obj.create_time
    self.duration = hunt_obj.duration
    self.creator = hunt_obj.creator
    self.description = hunt_obj.description
    self.is_robot = hunt_obj.creator in ["GRRWorker", "Cron"]
    if hunt_counters is not None:
      self.results_count = hunt_counters.num_results
      self.clients_with_results_count = hunt_counters.num_clients_with_results
      self.clients_queued_count = (
          hunt_counters.num_clients - hunt_counters.num_successful_clients -
          hunt_counters.num_failed_clients - hunt_counters.num_crashed_clients)
      # TODO(user): remove this hack when AFF4 is gone. For regression tests
      # compatibility only.
      self.total_cpu_usage = hunt_counters.total_cpu_seconds or 0
      self.total_net_usage = hunt_counters.total_network_bytes_sent

      if with_full_summary:
        self.all_clients_count = hunt_counters.num_clients
        self.completed_clients_count = (
            hunt_counters.num_successful_clients +
            hunt_counters.num_failed_clients)
        self.remaining_clients_count = (
            self.all_clients_count - self.completed_clients_count)
    else:
      self.results_count = 0
      self.clients_with_results_count = 0
      self.clients_queued_count = 0
      self.total_cpu_usage = 0
      self.total_net_usage = 0

      if with_full_summary:
        self.all_clients_count = 0
        self.completed_clients_count = 0
        self.remaining_clients_count = 0

    if hunt_obj.original_object.object_type != "UNKNOWN":
      ref = ApiFlowLikeObjectReference()
      self.original_object = ref.FromFlowLikeObjectReference(
          hunt_obj.original_object)

    if with_full_summary:
      hra = self.hunt_runner_args = rdf_hunts.HuntRunnerArgs(
          hunt_name=self.name,
          description=hunt_obj.description,
          client_rule_set=hunt_obj.client_rule_set,
          crash_limit=hunt_obj.crash_limit,
          avg_results_per_client_limit=hunt_obj.avg_results_per_client_limit,
          avg_cpu_seconds_per_client_limit=hunt_obj
          .avg_cpu_seconds_per_client_limit,
          avg_network_bytes_per_client_limit=hunt_obj
          .avg_network_bytes_per_client_limit,
          client_rate=hunt_obj.client_rate,
          original_object=hunt_obj.original_object)

      if hunt_obj.HasField("output_plugins"):
        hra.output_plugins = hunt_obj.output_plugins

      # TODO(user): This is a backwards compatibility code. Remove
      # HuntRunnerArgs from ApiHunt.
      if hunt_obj.client_limit != 100:
        hra.client_limit = hunt_obj.client_limit

      if hunt_obj.HasField("per_client_cpu_limit"):
        hra.per_client_cpu_limit = hunt_obj.per_client_cpu_limit

      if hunt_obj.HasField("per_client_network_limit_bytes"):
        hra.per_client_network_limit_bytes = (
            hunt_obj.per_client_network_bytes_limit)

      if hunt_obj.HasField("total_network_bytes_limit"):
        hra.network_bytes_limit = hunt_obj.total_network_bytes_limit

      self.client_rule_set = hunt_obj.client_rule_set

      if (hunt_obj.args.hunt_type ==
          rdf_hunt_objects.HuntArguments.HuntType.STANDARD):
        self.flow_name = hunt_obj.args.standard.flow_name
        self.flow_args = hunt_obj.args.standard.flow_args

    return self


Instruction:
Make a summary of the following Python 3 code
def InitFromFlowObject(self, fo):

    self.client_id = fo.client_id
    if fo.HasField("backtrace"):
      self.backtrace = fo.backtrace
    self.log_message = fo.error_message
    self.timestamp = fo.last_update_time

    return self

Response: Initialize from rdf_flow_objects. Flow object.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def InitFromDataPoints(self, start_stats, complete_stats):
    self.start_points = self._ConvertToResultList(start_stats)
    self.complete_points = self._ConvertToResultList(complete_stats)
    return self

Response: Initializes the approval from the given data points.


Instruction:
Can you generate the documentation for the following Python 3 function
def _Resample(self, stats, target_size):
    t_first = stats[0][0]
    t_last = stats[-1][0]
    interval = (t_last - t_first) / target_size

    result = []

    current_t = t_first
    current_v = 0
    i = 0
    while i < len(stats):
      stat_t = stats[i][0]
      stat_v = stats[i][1]
      if stat_t <= (current_t + interval):
        # Always add the last value in an interval to the result.
        current_v = stat_v
        i += 1
      else:
        result.append([current_t + interval, current_v])
        current_t += interval

    result.append([current_t + interval, current_v])
    return result

Response: Resamples the stats to have a specific number of data points.


Instruction:
How would you code a function in Python 3 to
retrieve the stats for a hunt.

Response:
def _HandleLegacy(self, args, token=None):
    """Retrieves the stats for a hunt."""
    hunt_obj = aff4.FACTORY.Open(
        args.hunt_id.ToURN(), aff4_type=implementation.GRRHunt, token=token)

    stats = hunt_obj.GetRunner().context.usage_stats

    return ApiGetHuntStatsResult(stats=stats)


Instruction:
How would you code a function in Python 3 to
retrieve the clients for a given hunt.

Response:
def _HandleLegacy(self, args, token=None):
    """Retrieves the clients for a hunt."""
    hunt_urn = args.hunt_id.ToURN()
    hunt_obj = aff4.FACTORY.Open(
        hunt_urn, aff4_type=implementation.GRRHunt, token=token)

    clients_by_status = hunt_obj.GetClientsByStatus()
    hunt_clients = clients_by_status[args.client_status.name]
    total_count = len(hunt_clients)

    if args.count:
      hunt_clients = sorted(hunt_clients)[args.offset:args.offset + args.count]
    else:
      hunt_clients = sorted(hunt_clients)[args.offset:]

    flow_id = "%s:hunt" % hunt_urn.Basename()
    results = [
        ApiHuntClient(client_id=c.Basename(), flow_id=flow_id)
        for c in hunt_clients
    ]
    return ApiListHuntClientsResult(items=results, total_count=total_count)


Instruction:
Can you tell what is the following Python 3 function doing
def _HandleLegacy(self, args, token=None):

    hunt_obj = aff4.FACTORY.Open(
        args.hunt_id.ToURN(), aff4_type=implementation.GRRHunt, token=token)

    if isinstance(hunt_obj.context, rdf_hunts.HuntContext):  # New style hunt.
      # TODO(amoser): Hunt state will go away soon, we don't render it anymore.
      state = api_call_handler_utils.ApiDataObject()
      result = ApiGetHuntContextResult(context=hunt_obj.context, state=state)
      # Assign args last since it needs the other fields set to
      # determine the args protobuf.
      result.args = hunt_obj.args
      return result

    else:
      # Just pack the whole context data object in the state
      # field. This contains everything for old style hunts so we at
      # least show the data somehow.
      context = api_call_handler_utils.ApiDataObject().InitFromDataObject(
          hunt_obj.context)

      return ApiGetHuntContextResult(state=context)

Response: Retrieves the context for a hunt.


Instruction:
Explain what the following Python 3 code does
def _HandleLegacy(self, args, token=None):

    # We only create generic hunts with /hunts/create requests.
    generic_hunt_args = rdf_hunts.GenericHuntArgs()
    generic_hunt_args.flow_runner_args.flow_name = args.flow_name
    generic_hunt_args.flow_args = args.flow_args

    # Clear all fields marked with HIDDEN, except for output_plugins - they are
    # marked HIDDEN, because we have a separate UI for them, not because they
    # shouldn't be shown to the user at all.
    #
    # TODO(user): Refactor the code to remove the HIDDEN label from
    # HuntRunnerArgs.output_plugins.
    args.hunt_runner_args.ClearFieldsWithLabel(
        rdf_structs.SemanticDescriptor.Labels.HIDDEN,
        exceptions="output_plugins")
    args.hunt_runner_args.hunt_name = standard.GenericHunt.__name__

    if args.original_hunt and args.original_flow:
      raise ValueError(
          "A hunt can't be a copy of a flow and a hunt at the same time.")
    if args.original_hunt:
      ref = rdf_hunts.FlowLikeObjectReference.FromHuntId(
          utils.SmartStr(args.original_hunt.hunt_id))
      args.hunt_runner_args.original_object = ref
    elif args.original_flow:
      ref = rdf_hunts.FlowLikeObjectReference.FromFlowIdAndClientId(
          utils.SmartStr(args.original_flow.flow_id),
          utils.SmartStr(args.original_flow.client_id))
      args.hunt_runner_args.original_object = ref

    # Anyone can create the hunt but it will be created in the paused
    # state. Permissions are required to actually start it.
    with implementation.StartHunt(
        runner_args=args.hunt_runner_args, args=generic_hunt_args,
        token=token) as hunt_obj:

      # Nothing really to do here - hunts are always created in the paused
      # state.
      logging.info("User %s created a new %s hunt (%s)", token.username,
                   hunt_obj.args.flow_runner_args.flow_name, hunt_obj.urn)

      return ApiHunt().InitFromAff4Object(hunt_obj, with_full_summary=True)

Response: Handles the legacy hunt creation and creation of a new hunt.


Instruction:
Write a Python 3 script to
register a new constructor in the internal list of constructors.

Response:
def Register(self, name, constructor):
    """Registers a new constructor in the factory.

    Args:
      name: A name associated with given constructor.
      constructor: A constructor function that creates instances.

    Raises:
      ValueError: If there already is a constructor associated with given name.
    """
    precondition.AssertType(name, Text)

    if name in self._constructors:
      message = "Duplicated constructors %r and %r for name '%s'"
      message %= (constructor, self._constructors[name], name)
      raise ValueError(message)

    self._constructors[name] = constructor


Instruction:
Can you tell what is the following Python 3 function doing
def Unregister(self, name):
    precondition.AssertType(name, Text)

    try:
      del self._constructors[name]
    except KeyError:
      raise ValueError("Constructor with name '%s' is not registered" % name)

Response: Unregisters a constructor with the specified name.


Instruction:
Can you implement a function in Python 3 that
creates a new instance of the type that the factory supports.

Response:
def Create(self, name):
    """Creates a new instance.

    Args:
      name: A name identifying the constructor to use for instantiation.

    Returns:
      An instance of the type that the factory supports.
    """
    precondition.AssertType(name, Text)

    try:
      constructor = self._constructors[name]
    except KeyError:
      message = "No constructor for name '%s' has been registered"
      message %= name
      raise ValueError(message)

    instance = constructor()
    if not isinstance(instance, self._cls):
      message = ("Constructor %r for name '%s' returned instance of type %r "
                 "(expected %r)")
      message %= (constructor, name, type(instance), self._cls)
      raise TypeError(message)

    return instance


Instruction:
Can you write a function in Python 3 where it
registers a stat entry at a given timestamp.

Response:
def AddStatEntry(self, stat_entry, timestamp):
    """Registers stat entry at a given timestamp."""

    if timestamp in self._stat_entries:
      message = ("Duplicated stat entry write for path '%s' of type '%s' at "
                 "timestamp '%s'. Old: %s. New: %s.")
      message %= ("/".join(self._components), self._path_type, timestamp,
                  self._stat_entries[timestamp], stat_entry)
      raise db.Error(message)

    if timestamp not in self._path_infos:
      path_info = rdf_objects.PathInfo(
          path_type=self._path_type,
          components=self._components,
          timestamp=timestamp,
          stat_entry=stat_entry)
      self.AddPathInfo(path_info)
    else:
      self._path_infos[timestamp].stat_entry = stat_entry


Instruction:
How would you implement a function in Python 3 that
registers a hash entry at a given timestamp.

Response:
def AddHashEntry(self, hash_entry, timestamp):
    """Registers hash entry at a given timestamp."""

    if timestamp in self._hash_entries:
      message = ("Duplicated hash entry write for path '%s' of type '%s' at "
                 "timestamp '%s'. Old: %s. New: %s.")
      message %= ("/".join(self._components), self._path_type, timestamp,
                  self._hash_entries[timestamp], hash_entry)
      raise db.Error(message)

    if timestamp not in self._path_infos:
      path_info = rdf_objects.PathInfo(
          path_type=self._path_type,
          components=self._components,
          timestamp=timestamp,
          hash_entry=hash_entry)
      self.AddPathInfo(path_info)
    else:
      self._path_infos[timestamp].hash_entry = hash_entry


Instruction:
Here you have a function in Python 3, explain what it does
def AddPathInfo(self, path_info):

    if self._path_type != path_info.path_type:
      message = "Incompatible path types: `%s` and `%s`"
      raise ValueError(message % (self._path_type, path_info.path_type))
    if self._components != path_info.components:
      message = "Incompatible path components: `%s` and `%s`"
      raise ValueError(message % (self._components, path_info.components))

    if path_info.timestamp in self._path_infos:
      raise ValueError("PathInfo with timestamp %r was added before." %
                       path_info.timestamp)

    new_path_info = path_info.Copy()
    if new_path_info.timestamp is None:
      new_path_info.timestamp = rdfvalue.RDFDatetime.Now()
    self._path_infos[new_path_info.timestamp] = new_path_info

Response: Adds a path info to the path record.


Instruction:
How would you explain what the following Python 3 function does
def AddChild(self, path_info):

    if self._path_type != path_info.path_type:
      message = "Incompatible path types: `%s` and `%s`"
      raise ValueError(message % (self._path_type, path_info.path_type))
    if self._components != path_info.components[:-1]:
      message = "Incompatible path components, expected `%s` but got `%s`"
      raise ValueError(message % (self._components, path_info.components[:-1]))

    self._children.add(path_info.GetPathID())

Response: Makes the path aware of some child.


Instruction:
Create a Python 3 function to
generate a summary about the path record.

Response:
def GetPathInfo(self, timestamp=None):
    """Generates a summary about the path record.

    Args:
      timestamp: A point in time from which the data should be retrieved.

    Returns:
      A `rdf_objects.PathInfo` instance.
    """
    path_info_timestamp = self._LastEntryTimestamp(self._path_infos, timestamp)
    try:
      result = self._path_infos[path_info_timestamp].Copy()
    except KeyError:
      result = rdf_objects.PathInfo(
          path_type=self._path_type, components=self._components)

    stat_entry_timestamp = self._LastEntryTimestamp(self._stat_entries,
                                                    timestamp)
    result.last_stat_entry_timestamp = stat_entry_timestamp
    result.stat_entry = self._stat_entries.get(stat_entry_timestamp)

    hash_entry_timestamp = self._LastEntryTimestamp(self._hash_entries,
                                                    timestamp)
    result.last_hash_entry_timestamp = hash_entry_timestamp
    result.hash_entry = self._hash_entries.get(hash_entry_timestamp)

    return result


Instruction:
Given the following Python 3 function, write the documentation
def _LastEntryTimestamp(dct, upper_bound_timestamp):
    if upper_bound_timestamp is None:
      upper_bound = lambda _: True
    else:
      upper_bound = lambda key: key <= upper_bound_timestamp

    try:
      return max(filter(upper_bound, iterkeys(dct)))
    except ValueError:  # Thrown if `max` input (result of filtering) is empty.
      return None

Response: Searches for the highest timestamp lower than the specified one.


Instruction:
Can you generate the documentation for the following Python 3 function
def ReadPathInfo(self, client_id, path_type, components, timestamp=None):
    try:
      path_record = self.path_records[(client_id, path_type, components)]
      return path_record.GetPathInfo(timestamp=timestamp)
    except KeyError:
      raise db.UnknownPathError(
          client_id=client_id, path_type=path_type, components=components)

Response: Reads a path info record for a given path.


Instruction:
How would you code a function in Python 3 to
read path info records for given paths.

Response:
def ReadPathInfos(self, client_id, path_type, components_list):
    """Retrieves path info records for given paths."""
    result = {}

    for components in components_list:
      try:
        path_record = self.path_records[(client_id, path_type, components)]
        result[components] = path_record.GetPathInfo()
      except KeyError:
        result[components] = None

    return result


Instruction:
Can you generate the documentation for the following Python 3 function
def ListDescendentPathInfos(self,
                              client_id,
                              path_type,
                              components,
                              timestamp=None,
                              max_depth=None):
    """Lists path info records that correspond to children of given path."""
    result = []

    for path_idx, path_record in iteritems(self.path_records):
      other_client_id, other_path_type, other_components = path_idx
      if client_id != other_client_id or path_type != other_path_type:
        continue
      if len(other_components) == len(components):
        continue
      if not collection.StartsWith(other_components, components):
        continue
      if (max_depth is not None and
          len(other_components) - len(components) > max_depth):
        continue

      result.append(path_record.GetPathInfo(timestamp=timestamp))

    if timestamp is None:
      return sorted(result, key=lambda _: tuple(_.components))

    # We need to filter implicit path infos if specific timestamp is given.

    # TODO(hanuszczak): If we were to switch to use path trie instead of storing
    # records by path id, everything would be much easier.

    class TrieNode(object):
      """A trie of path components with path infos as values."""

      def __init__(self):
        self.path_info = None
        self.children = {}
        self.explicit = False

      def Add(self, path_info, idx=0):
        """Adds given path info to the trie (or one of its subtrees)."""
        components = path_info.components
        if idx == len(components):
          self.path_info = path_info
          self.explicit |= (
              path_info.HasField("stat_entry") or
              path_info.HasField("hash_entry"))
        else:
          child = self.children.setdefault(components[idx], TrieNode())
          child.Add(path_info, idx=idx + 1)
          self.explicit |= child.explicit

      def Collect(self, path_infos):
        if self.path_info is not None and self.explicit:
          path_infos.append(self.path_info)

        for component in sorted(iterkeys(self.children)):
          self.children[component].Collect(path_infos)

    trie = TrieNode()
    for path_info in result:
      trie.Add(path_info)

    explicit_path_infos = []
    trie.Collect(explicit_path_infos)
    return explicit_path_infos

Response: Lists all path info records that correspond to children of given path.


Instruction:
Here you have a function in Python 3, explain what it does
def _WritePathInfo(self, client_id, path_info):
    if client_id not in self.metadatas:
      raise db.UnknownClientError(client_id)

    path_record = self._GetPathRecord(client_id, path_info)
    path_record.AddPathInfo(path_info)

    parent_path_info = path_info.GetParent()
    if parent_path_info is not None:
      parent_path_record = self._GetPathRecord(client_id, parent_path_info)
      parent_path_record.AddChild(path_info)

Response: Writes a single path info record for given client.


Instruction:
How would you implement a function in Python 3 that
clears the path history for the given paths of given clients.

Response:
def MultiClearPathHistory(self, path_infos):
    """Clears path history for specified paths of given clients."""
    for client_id, client_path_infos in iteritems(path_infos):
      self.ClearPathHistory(client_id, client_path_infos)


Instruction:
Can you tell what is the following Python 3 function doing
def ClearPathHistory(self, client_id, path_infos):
    for path_info in path_infos:
      path_record = self._GetPathRecord(client_id, path_info)
      path_record.ClearHistory()

Response: Clears the path history for the given paths of given client.


Instruction:
Can you generate the documentation for the following Python 3 function
def MultiWritePathHistory(self, client_path_histories):
    for client_path, client_path_history in iteritems(client_path_histories):
      if client_path.client_id not in self.metadatas:
        raise db.UnknownClientError(client_path.client_id)

      path_info = rdf_objects.PathInfo(
          path_type=client_path.path_type, components=client_path.components)

      for timestamp, stat_entry in iteritems(client_path_history.stat_entries):
        path_record = self._GetPathRecord(
            client_path.client_id, path_info, set_default=False)
        if path_record is None:
          # TODO(hanuszczak): Provide more details about paths that caused that.
          raise db.AtLeastOneUnknownPathError([])

        path_record.AddStatEntry(stat_entry, timestamp)

      for timestamp, hash_entry in iteritems(client_path_history.hash_entries):
        path_record = self._GetPathRecord(
            client_path.client_id, path_info, set_default=False)
        if path_record is None:
          # TODO(hanuszczak): Provide more details about paths that caused that.
          raise db.AtLeastOneUnknownPathError([])

        path_record.AddHashEntry(hash_entry, timestamp)

Response: Writes a collection of hash and stat entries observed for given paths.


Instruction:
Implement a function in Python 3 to
read a collection of hash and stat entries for given paths.

Response:
def ReadPathInfosHistories(self, client_id, path_type, components_list):
    """Reads a collection of hash and stat entries for given paths."""
    results = {}

    for components in components_list:
      try:
        path_record = self.path_records[(client_id, path_type, components)]
      except KeyError:
        results[components] = []
        continue

      entries_by_ts = {}
      for ts, stat_entry in path_record.GetStatEntries():
        pi = rdf_objects.PathInfo(
            path_type=path_type,
            components=components,
            timestamp=ts,
            stat_entry=stat_entry)
        entries_by_ts[ts] = pi

      for ts, hash_entry in path_record.GetHashEntries():
        try:
          pi = entries_by_ts[ts]
        except KeyError:
          pi = rdf_objects.PathInfo(
              path_type=path_type, components=components, timestamp=ts)
          entries_by_ts[ts] = pi

        pi.hash_entry = hash_entry

      results[components] = [
          entries_by_ts[k] for k in sorted(iterkeys(entries_by_ts))
      ]

    return results


Instruction:
Can you write a function in Python 3 where it
converts the supported_os property into a condition.

Response:
def ConvertSupportedOSToConditions(src_object):
  """Turn supported_os into a condition."""
  if src_object.supported_os:
    conditions = " OR ".join("os == '%s'" % o for o in src_object.supported_os)
    return conditions


Instruction:
Can you tell what is the following Python 3 function doing
def GetArtifactCollectorArgs(flow_args, knowledge_base):
  args = rdf_artifacts.ClientArtifactCollectorArgs()
  args.knowledge_base = knowledge_base

  args.apply_parsers = flow_args.apply_parsers
  args.ignore_interpolation_errors = flow_args.ignore_interpolation_errors
  args.max_file_size = flow_args.max_file_size
  args.use_tsk = flow_args.use_tsk

  if not flow_args.recollect_knowledge_base:
    artifact_names = flow_args.artifact_list
  else:
    artifact_names = GetArtifactsForCollection(knowledge_base.os,
                                               flow_args.artifact_list)

  expander = ArtifactExpander(knowledge_base, flow_args.path_type,
                              flow_args.max_file_size)
  for artifact_name in artifact_names:
    rdf_artifact = artifact_registry.REGISTRY.GetArtifact(artifact_name)
    if not MeetsConditions(knowledge_base, rdf_artifact):
      continue
    if artifact_name in expander.processed_artifacts:
      continue
    requested_by_user = artifact_name in flow_args.artifact_list
    for expanded_artifact in expander.Expand(rdf_artifact, requested_by_user):
      args.artifacts.append(expanded_artifact)
  return args

Response: Prepare bundle of artifacts and their dependencies for the client.


Instruction:
Can you implement a function in Python 3 that
checks conditions on the source.

Response:
def MeetsConditions(knowledge_base, source):
  """Check conditions on the source."""
  source_conditions_met = True
  os_conditions = ConvertSupportedOSToConditions(source)
  if os_conditions:
    source.conditions.append(os_conditions)
  for condition in source.conditions:
    source_conditions_met &= artifact_utils.CheckCondition(
        condition, knowledge_base)

  return source_conditions_met


Instruction:
Can you generate a brief explanation for the following Python 3 code
def GetArtifactsForCollection(os_name, artifact_list):
  artifact_arranger = ArtifactArranger(os_name, artifact_list)
  artifact_names = artifact_arranger.GetArtifactsInProperOrder()
  return artifact_names

Response: Wrapper for ArtifactArranger. GetArtifactsForCollection.


Instruction:
Explain what the following Python 3 code does
def Expand(self, rdf_artifact, requested):

    source_type = rdf_artifacts.ArtifactSource.SourceType

    expanded_artifact = rdf_artifacts.ExpandedArtifact(
        name=rdf_artifact.name,
        provides=rdf_artifact.provides,
        requested_by_user=requested)

    for source in rdf_artifact.sources:
      if MeetsConditions(self._knowledge_base, source):
        type_name = source.type

        if type_name == source_type.ARTIFACT_GROUP:
          for subartifact in self._ExpandArtifactGroupSource(source, requested):
            yield subartifact
          continue

        elif type_name == source_type.ARTIFACT_FILES:
          expanded_sources = self._ExpandArtifactFilesSource(source, requested)

        else:
          expanded_sources = self._ExpandBasicSource(source)

        expanded_artifact.sources.Extend(expanded_sources)
    self.processed_artifacts.add(rdf_artifact.name)
    if expanded_artifact.sources:
      yield expanded_artifact

Response: Expand an artifact by extending its sources.


Instruction:
Here you have a function in Python 3, explain what it does
def _ExpandArtifactGroupSource(self, source, requested):
    artifact_list = []
    if "names" in source.attributes:
      artifact_list = source.attributes["names"]
    for artifact_name in artifact_list:
      if artifact_name in self.processed_artifacts:
        continue
      artifact_obj = artifact_registry.REGISTRY.GetArtifact(artifact_name)
      for expanded_artifact in self.Expand(artifact_obj, requested):
        yield expanded_artifact

Response: Recursively expands an artifact group source.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _ExpandArtifactFilesSource(self, source, requested):
    expanded_source = rdf_artifacts.ExpandedSource(base_source=source)
    sub_sources = []
    artifact_list = []
    if "artifact_list" in source.attributes:
      artifact_list = source.attributes["artifact_list"]
    for artifact_name in artifact_list:
      if artifact_name in self.processed_artifacts:
        continue
      artifact_obj = artifact_registry.REGISTRY.GetArtifact(artifact_name)
      for expanded_artifact in self.Expand(artifact_obj, requested):
        sub_sources.extend(expanded_artifact.sources)
    expanded_source.artifact_sources = sub_sources
    expanded_source.path_type = self._path_type
    return [expanded_source]

Response: Recursively expands an artifact files source.


Instruction:
Create a Python 3 function for
initializing the dependency graph.

Response:
def _InitializeGraph(self, os_name, artifact_list):
    """Creates the nodes and directed edges of the dependency graph.

    Args:
      os_name: String specifying the OS name.
      artifact_list: List of requested artifact names.
    """
    dependencies = artifact_registry.REGISTRY.SearchDependencies(
        os_name, artifact_list)
    artifact_names, attribute_names = dependencies

    self._AddAttributeNodes(attribute_names)
    self._AddArtifactNodesAndEdges(artifact_names)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _AddAttributeNodes(self, attribute_names):
    for attribute_name in attribute_names:
      self.graph[attribute_name] = self.Node(is_artifact=False)

Response: Adds the attribute nodes to the dependency graph.


Instruction:
Create a Python 3 function for
adding the artifact nodes and edges to the graph.

Response:
def _AddArtifactNodesAndEdges(self, artifact_names):
    """Add the artifact nodes to the graph.

    For every artifact that has to be collected, add a node to the dependency
    graph.

    The edges represent the dependencies. An artifact has outgoing edges to the
    attributes it provides and incoming edges from attributes it depends on.
    Initially, only artifacts without incoming edges are reachable. An artifact
    becomes reachable if all of its dependencies are reachable.

    Args:
      artifact_names: List of names of the artifacts to collect.
    """
    for artifact_name in artifact_names:
      self.graph[artifact_name] = self.Node(is_artifact=True)
      rdf_artifact = artifact_registry.REGISTRY.GetArtifact(artifact_name)
      self._AddDependencyEdges(rdf_artifact)
      self._AddProvidesEdges(rdf_artifact)


Instruction:
Create a Python 3 function to
add an edge for every dependency of the given artifact.

Response:
def _AddDependencyEdges(self, rdf_artifact):
    """Add an edge for every dependency of the given artifact.

    This method gets the attribute names for a given artifact and for every
    attribute it adds a directed edge from the attribute node to the artifact
    node. If an artifact does not have any dependencies it is added to the set
    of reachable nodes.

    Args:
      rdf_artifact: The artifact object.
    """
    artifact_dependencies = artifact_registry.GetArtifactPathDependencies(
        rdf_artifact)
    if artifact_dependencies:
      for attribute in artifact_dependencies:
        self._AddEdge(attribute, rdf_artifact.name)
    else:
      self.reachable_nodes.add(rdf_artifact.name)
      self.graph[rdf_artifact.name].is_provided = True


Instruction:
Implement a function in Python 3 to
add an edge for every attribute in the given artifact provides.

Response:
def _AddProvidesEdges(self, rdf_artifact):
    """Add an edge for every attribute the given artifact provides.

    This method adds a directed edge from the artifact node to every attribute
    this artifact provides.

    Args:
      rdf_artifact: The artifact object.
    """
    for attribute in rdf_artifact.provides:
      self._AddEdge(rdf_artifact.name, attribute)


Instruction:
Can you generate the documentation for the following Python 3 function
def _AddEdge(self, start_node, end_node):

    self.graph[start_node].outgoing.append(end_node)

    # This check is necessary because an artifact can provide attributes that
    # are not covered by the graph because they are not relevant for the
    # requested artifacts.
    if end_node in self.graph:
      self.graph[end_node].incoming.append(start_node)

Response: Adds a directed edge to the graph.


Instruction:
Create a Python 3 function to
bring the artifacts in a linear order that resolves dependencies.

Response:
def GetArtifactsInProperOrder(self):
    """Bring the artifacts in a linear order that resolves dependencies.

    This method obtains a linear ordering of the nodes and then returns the list
    of artifact names.

    Returns:
      A list of `ArtifactName` instances such that if they are collected in the
      given order their dependencies are resolved.
    """
    artifact_list = []
    while self.reachable_nodes:
      node_name = self.reachable_nodes.pop()
      node = self.graph[node_name]
      if node.is_artifact:
        artifact_list.append(node_name)
      for next_node_name in node.outgoing:
        if next_node_name not in self.graph:
          continue
        next_node = self.graph[next_node_name]
        if next_node.is_provided:
          continue
        next_node.incoming.remove(node_name)
        if not (next_node.is_artifact and next_node.incoming):
          next_node.is_provided = True
          self.reachable_nodes.add(next_node_name)
    return artifact_list


Instruction:
How would you code a function in Python 3 to
consume an entire range or part thereof.

Response:
def ConsumeRange(self, start, end):
    """Consumes an entire range, or part thereof.

    If the finger has no ranges left, or the curent range start is higher
    than the end of the consumed block, nothing happens. Otherwise,
    the current range is adjusted for the consumed block, or removed,
    if the entire block is consumed. For things to work, the consumed
    range and the current finger starts must be equal, and the length
    of the consumed range may not exceed the length of the current range.

    Args:
      start: Beginning of range to be consumed.
      end: First offset after the consumed range (end + 1).

    Raises:
      RuntimeError: if the start position of the consumed range is
          higher than the start of the current range in the finger, or if
          the consumed range cuts accross block boundaries.
    """
    old = self.CurrentRange()
    if old is None:
      return
    if old.start > start:
      if old.start < end:
        raise RuntimeError('Block end too high.')
      return
    if old.start < start:
      raise RuntimeError('Block start too high.')
    if old.end == end:
      del self.ranges[0]
    elif old.end > end:
      self.ranges[0] = Range(end, old.end)
    else:
      raise RuntimeError('Block length exceeds range.')


Instruction:
How would you code a function in Python 3 to
return the next range of interest in a namedtuple.

Response:
def _GetNextInterval(self):
    """Returns the next Range of the file that is to be hashed.

    For all fingers, inspect their next expected range, and return the
    lowest uninterrupted range of interest. If the range is larger than
    BLOCK_SIZE, truncate it.

    Returns:
      Next range of interest in a Range namedtuple.
    """
    ranges = [x.CurrentRange() for x in self.fingers]
    starts = set([r.start for r in ranges if r])
    ends = set([r.end for r in ranges if r])
    if not starts:
      return None
    min_start = min(starts)
    starts.remove(min_start)
    ends |= starts
    min_end = min(ends)
    if min_end - min_start > self.BLOCK_SIZE:
      min_end = min_start + self.BLOCK_SIZE
    return Range(min_start, min_end)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _HashBlock(self, block, start, end):
    for finger in self.fingers:
      expected_range = finger.CurrentRange()
      if expected_range is None:
        continue
      if (start > expected_range.start or
          (start == expected_range.start and end > expected_range.end) or
          (start < expected_range.start and end > expected_range.start)):
        raise RuntimeError('Cutting across fingers.')
      if start == expected_range.start:
        finger.HashBlock(block)

Response: Hashes the given block into the hashers of the fingers.


Instruction:
How would you explain what the following Python 3 function does
def HashIt(self):
    while True:
      interval = self._GetNextInterval()
      if interval is None:
        break
      self.file.seek(interval.start, os.SEEK_SET)
      block = self.file.read(interval.end - interval.start)
      if len(block) != interval.end - interval.start:
        raise RuntimeError('Short read on file.')
      self._HashBlock(block, interval.start, interval.end)
      self._AdjustIntervals(interval.start, interval.end)

    results = []
    for finger in self.fingers:
      res = {}
      leftover = finger.CurrentRange()
      if leftover:
        if (len(finger.ranges) > 1 or leftover.start != self.filelength or
            leftover.end != self.filelength):
          raise RuntimeError('Non-empty range remains.')
      res.update(finger.metadata)
      for hasher in finger.hashers:
        res[hasher.name] = hasher.digest()
      results.append(res)

    # Clean out things for a fresh start (on the same file object).
    self.fingers = []

    # Make sure the results come back in 'standard' order, regardless of the
    # order in which fingers were added. Helps with reproducing test results.
    return sorted(results, key=lambda r: r['name'])

Response: Finalizing function for the Fingerprint class. This method applies all the different hash functions over the input file and computes the resulting hashes.


Instruction:
How would you explain what the following Python 3 function does
def EvalGeneric(self, hashers=None):
    if hashers is None:
      hashers = Fingerprinter.GENERIC_HASH_CLASSES
    hashfuncs = [x() for x in hashers]
    finger = Finger(hashfuncs, [Range(0, self.filelength)], {'name': 'generic'})
    self.fingers.append(finger)
    return True

Response: Evaluates the entire file with the given hash functions.


Instruction:
Write a Python 3 function for
parsing PECOFF headers. Reads header magic and some data structures in a file to determine if it is a valid PECOFF header, and figure out the offsets at which relevant data is stored. While this code contains multiple seeks and small reads, that is compensated by the underlying libc buffering mechanism. Returns: None if the parsed file is not PECOFF. A dict with offsets and lengths for CheckSum, CertTable, and SignedData fields in the PECOFF binary, for those that are present.

Response:
def _PecoffHeaderParser(self):
    """Parses PECOFF headers.

    Reads header magic and some data structures in a file to determine if
    it is a valid PECOFF header, and figure out the offsets at which
    relevant data is stored.
    While this code contains multiple seeks and small reads, that is
    compensated by the underlying libc buffering mechanism.

    Returns:
      None if the parsed file is not PECOFF.
      A dict with offsets and lengths for CheckSum, CertTable, and SignedData
      fields in the PECOFF binary, for those that are present.
    """
    extents = {}
    self.file.seek(0, os.SEEK_SET)
    buf = self.file.read(2)
    if buf != b'MZ':
      return None
    self.file.seek(0x3C, os.SEEK_SET)
    buf = self.file.read(4)
    pecoff_sig_offset = struct.unpack('<I', buf)[0]
    if pecoff_sig_offset >= self.filelength:
      return None
    self.file.seek(pecoff_sig_offset, os.SEEK_SET)
    buf = self.file.read(4)
    if buf != b'PE\0\0':
      return None
    self.file.seek(pecoff_sig_offset + 20, os.SEEK_SET)
    buf = self.file.read(2)
    optional_header_size = struct.unpack('<H', buf)[0]
    optional_header_offset = pecoff_sig_offset + 4 + 20
    if optional_header_size + optional_header_offset > self.filelength:
      # This is not strictly a failure for windows, but such files better
      # be treated as generic files. They can not be carrying SignedData.
      return None
    if optional_header_size < 68:
      # We can't do authenticode-style hashing. If this is a valid binary,
      # which it can be, the header still does not even contain a checksum.
      return None
    self.file.seek(optional_header_offset, os.SEEK_SET)
    buf = self.file.read(2)
    image_magic = struct.unpack('<H', buf)[0]
    if image_magic == 0x10b:
      # 32 bit
      rva_base = optional_header_offset + 92
      cert_base = optional_header_offset + 128
    elif image_magic == 0x20b:
      # 64 bit
      rva_base = optional_header_offset + 108
      cert_base = optional_header_offset + 144
    else:
      # A ROM image or such, not in the PE/COFF specs. Not sure what to do.
      return None
    extents['CheckSum'] = RelRange(optional_header_offset + 64, 4)
    self.file.seek(rva_base, os.SEEK_SET)
    buf = self.file.read(4)
    number_of_rva = struct.unpack('<I', buf)[0]
    if (number_of_rva < 5 or
        optional_header_offset + optional_header_size < cert_base + 8):
      return extents
    extents['CertTable'] = RelRange(cert_base, 8)

    self.file.seek(cert_base, os.SEEK_SET)
    buf = self.file.read(8)
    start, length = struct.unpack('<II', buf)
    if (length == 0 or start < optional_header_offset + optional_header_size or
        start + length > self.filelength):
      # The location of the SignedData blob is just wrong (or there is none).
      # Ignore it -- everything else we did still makes sense.
      return extents
    extents['SignedData'] = RelRange(start, length)
    return extents


Instruction:
Make a summary of the following Python 3 code
def _CollectSignedData(self, extent):
    start, length = extent

    self.file.seek(start, os.SEEK_SET)
    buf = self.file.read(length)
    signed_data = []
    # This loop ignores trailing cruft, or too-short signedData chunks.
    while len(buf) >= 8:
      dw_length, w_revision, w_cert_type = struct.unpack('<IHH', buf[:8])
      if dw_length < 8:
        # If the entire blob is smaller than its header, bail out.
        return signed_data
      b_cert = buf[8:dw_length]
      buf = buf[(dw_length + 7) & 0x7ffffff8:]
      signed_data.append((w_revision, w_cert_type, b_cert))
    return signed_data

Response: Extracts signedData blob from PECOFF binary and parses first layer.


Instruction:
Explain what the following Python 3 code does
def EvalPecoff(self, hashers=None):
    try:
      extents = self._PecoffHeaderParser()
    except struct.error:
      # Parsing the header failed. Just ignore this, and claim
      # that the file is not a valid PE/COFF image file.
      extents = None
    if extents is None:
      return False

    signed_data = None
    ranges = []
    start = 0
    # Ordering of these conditions matches expected order in file.
    # If a condition holds true, the matching range is skipped for hashing.
    if 'CheckSum' in extents:
      ranges.append(Range(start, end=extents['CheckSum'].start))
      start = sum(extents['CheckSum'])
      # New start now points past CheckSum area.
    if 'CertTable' in extents:
      ranges.append(Range(start, end=extents['CertTable'].start))
      start = sum(extents['CertTable'])
      # New start now points past CertTable area.
    if 'SignedData' in extents:
      # Exclude the range even if the blob itself can't be parsed correctly.
      ranges.append(Range(start, end=extents['SignedData'].start))
      start = sum(extents['SignedData'])
      # New start now points past SignedData area.
      signed_data = self._CollectSignedData(extents['SignedData'])
    ranges.append(Range(start, end=self.filelength))

    if hashers is None:
      hashers = Fingerprinter.AUTHENTICODE_HASH_CLASSES
    hashfuncs = [x() for x in hashers]
    metadata = {'name': 'pecoff'}
    if signed_data:
      metadata['SignedData'] = signed_data
    finger = Finger(hashfuncs, ranges, metadata)
    self.fingers.append(finger)
    return True

Response: Evaluates the PE or COFF file.


Instruction:
Write a Python 3 script to
parse the History file.

Response:
def Parse(self, stat, file_object, knowledge_base):
    """Parse the History file."""
    _, _ = stat, knowledge_base
    # TODO(user): Convert this to use the far more intelligent plaso parser.
    ff = Firefox3History(file_object)
    for timestamp, unused_entry_type, url, title in ff.Parse():
      yield rdf_webhistory.BrowserHistoryItem(
          url=url,
          domain=urlparse.urlparse(url).netloc,
          access_time=timestamp,
          program_name="Firefox",
          source_path=file_object.Path(),
          title=title)


Instruction:
Can you tell what is the following Python 3 function doing
def Parse(self):
    for timestamp, url, title in self.Query(self.VISITS_QUERY):
      if not isinstance(timestamp, (long, int)):
        timestamp = 0

      yield [timestamp, "FIREFOX3_VISIT", url, title]

Response: Iterator returning dict for each entry in history.


Instruction:
Implement a function in Python 3 to
read a tag from the buffer and return a tuple.

Response:
def ReadTag(buf, pos):
  """Read a tag from the buffer, and return a (tag_bytes, new_pos) tuple."""
  try:
    start = pos
    while ORD_MAP_AND_0X80[buf[pos]]:
      pos += 1
    pos += 1
    return (buf[start:pos], pos)
  except IndexError:
    raise ValueError("Invalid tag")


Instruction:
Can you generate a brief explanation for the following Python 3 code
def VarintEncode(value):
  result = b""
  if value < 0:
    raise ValueError("Varint can not encode a negative number.")

  bits = value & 0x7f
  value >>= 7

  while value:
    result += HIGH_CHR_MAP[bits]
    bits = value & 0x7f
    value >>= 7

  result += CHR_MAP[bits]

  return result

Response: Convert an integer to a varint and write it using the write function.


Instruction:
Write a Python 3 function that can
encode a signed integer as a zigzag encoded signed integer.

Response:
def SignedVarintEncode(value):
  """Encode a signed integer as a zigzag encoded signed integer."""
  result = b""
  if value < 0:
    value += (1 << 64)

  bits = value & 0x7f
  value >>= 7
  while value:
    result += HIGH_CHR_MAP[bits]
    bits = value & 0x7f
    value >>= 7
  result += CHR_MAP[bits]

  return result


Instruction:
How would you explain what the following Python 3 function does
def VarintReader(buf, pos=0):
  result = 0
  shift = 0
  while 1:
    b = buf[pos]

    result |= (ORD_MAP_AND_0X7F[b] << shift)
    pos += 1
    if not ORD_MAP_AND_0X80[b]:
      return (result, pos)
    shift += 7
    if shift >= 64:
      raise rdfvalue.DecodeError("Too many bytes when decoding varint.")

Response: A 64 bit decoder from google. protobuf. internal. decoder.


Instruction:
Can you implement a function in Python 3 that
splits the buffer into a list of strings.

Response:
def SplitBuffer(buff, index=0, length=None):
  """Parses the buffer as a prototypes.

  Args:
    buff: The buffer to parse.
    index: The position to start parsing.
    length: Optional length to parse until.

  Yields:
    Splits the buffer into tuples of strings:
        (encoded_tag, encoded_length, wire_format).
  """
  buffer_len = length or len(buff)
  while index < buffer_len:
    # data_index is the index where the data begins (i.e. after the tag).
    encoded_tag, data_index = ReadTag(buff, index)

    tag_type = ORD_MAP[encoded_tag[0]] & TAG_TYPE_MASK
    if tag_type == WIRETYPE_VARINT:
      # new_index is the index of the next tag.
      _, new_index = VarintReader(buff, data_index)
      yield (encoded_tag, b"", buff[data_index:new_index])
      index = new_index

    elif tag_type == WIRETYPE_FIXED64:
      yield (encoded_tag, b"", buff[data_index:data_index + 8])
      index = 8 + data_index

    elif tag_type == WIRETYPE_FIXED32:
      yield (encoded_tag, b"", buff[data_index:data_index + 4])
      index = 4 + data_index

    elif tag_type == WIRETYPE_LENGTH_DELIMITED:
      # Start index of the string.
      length, start = VarintReader(buff, data_index)
      yield (
          encoded_tag,
          buff[data_index:start],  # Encoded length.
          buff[start:start + length])  # Raw data of element.
      index = start + length

    else:
      raise rdfvalue.DecodeError("Unexpected Tag.")


Instruction:
Can you tell what is the following Python 3 function doing
def _GetOrderedEntries(data):

  # The raw data dictionary has two kinds of keys: strings (which correspond to
  # field name) or integers (if the name is unknown). In Python 3 it is not
  # possible to compare integers and strings to each other, so we first tag each
  # with either a 0 or 1 (so named fields are going to be serialized first) and
  # let the lexicographical ordering of the tuples take care of the rest.
  def Tag(field):
    """Tags field name with a number to make comparison possible."""

    # TODO: We use `string_types` here because in Python 2
    # attribute names (which are passed e.g. through keyword arguments) are
    # represented as `bytes` whereas in Python 3 it is `unicode`. This should
    # be replaced with `str` once support for Python 2 is dropped.
    if isinstance(field, string_types):
      return 0, field
    if isinstance(field, int):
      return 1, field

    message = "Unexpected field '{}' of type '{}'".format(field, type(field))
    raise TypeError(message)

  for field in sorted(iterkeys(data), key=Tag):
    yield data[field]

Response: Returns an iterator over the entries of the RDFProtoStruct in a well - defined order.


Instruction:
Can you write a function in Python 3 where it
serializes given triplets of python and wire values and a descriptor.

Response:
def _SerializeEntries(entries):
  """Serializes given triplets of python and wire values and a descriptor."""

  output = []
  for python_format, wire_format, type_descriptor in entries:

    if wire_format is None or (python_format and
                               type_descriptor.IsDirty(python_format)):
      wire_format = type_descriptor.ConvertToWireFormat(python_format)

    precondition.AssertIterableType(wire_format, bytes)
    output.extend(wire_format)

  return b"".join(output)


Instruction:
Create a Python 3 function for
reading all tags until the next end group and store in the value_obj.

Response:
def ReadIntoObject(buff, index, value_obj, length=0):
  """Reads all tags until the next end group and store in the value_obj."""
  raw_data = value_obj.GetRawData()
  count = 0

  # Split the buffer into tags and wire_format representations, then collect
  # these into the raw data cache.
  for (encoded_tag, encoded_length, encoded_field) in SplitBuffer(
      buff, index=index, length=length):

    type_info_obj = value_obj.type_infos_by_encoded_tag.get(encoded_tag)

    # Internal format to store parsed fields.
    wire_format = (encoded_tag, encoded_length, encoded_field)

    # If the tag is not found we need to skip it. Skipped fields are
    # inaccessible to this actual object, because they have no type info
    # describing them, however they are still stored in the raw data
    # representation because they will be re-serialized back. This way
    # programs which simply read protobufs and write them back do not need to
    # know all the fields, some of which were defined in a later version of
    # the application. In order to avoid having to worry about repeated fields
    # here, we just insert them into the raw data dict with a key which should
    # be unique.
    if type_info_obj is None:
      # Record an unknown field. The key is unique and ensures we do not collide
      # the dict on repeated fields of the encoded tag. Note that this field is
      # not really accessible using Get() and does not have a python format
      # representation. It will be written back using the same wire format it
      # was read with, therefore does not require a type descriptor at all.
      raw_data[count] = (None, wire_format, None)

      count += 1

    # Repeated fields are handled especially.
    elif type_info_obj.__class__ is ProtoList:
      value_obj.Get(type_info_obj.name).wrapped_list.append((None, wire_format))

    else:
      # Set the python_format as None so it gets converted lazily on access.
      raw_data[type_info_obj.name] = (None, wire_format, type_info_obj)

  value_obj.SetRawData(raw_data)


Instruction:
Given the following Python 3 function, write the documentation
def Validate(self, value, **_):
    if isinstance(value, rdfvalue.RDFString):
      # TODO(hanuszczak): Use `str` here.
      return Text(value)

    if isinstance(value, Text):
      return value

    if isinstance(value, bytes):
      return value.decode("utf-8")

    raise type_info.TypeValueError("Not a valid unicode string: %r" % value)

Response: Validates a python format representation of the value.


Instruction:
Make a summary of the following Python 3 code
def ConvertToWireFormat(self, value):
    value = value.encode("utf8")
    return (self.encoded_tag, VarintEncode(len(value)), value)

Response: Converts the value to wire format.


Instruction:
Can you implement a function in Python 3 that
checks that value is a valid enum value.

Response:
def Validate(self, value, **_):
    """Check that value is a valid enum."""
    # None is a valid value - it means the field is not set.
    if value is None:
      return

    # If the value is a string we need to try to convert it to an integer.
    checked_value = value
    if isinstance(value, string_types):
      # NOTE: that when initializing from string, enum values are
      # case-insensitive.
      checked_value = self.enum.get(value.upper())
      if checked_value is None and value.isdigit():
        checked_value = int(value)
      if checked_value is None:
        raise type_info.TypeValueError(
            "Value %s is not a valid enum value for field %s" %
            (value, self.name))

    return EnumNamedValue(checked_value, name=self.reverse_enum.get(value))


Instruction:
Can you tell what is the following Python 3 function doing
def Definition(self):
    result = self._FormatDescriptionComment()

    result += "  enum %s {\n" % self.enum_name
    for k, v in sorted(iteritems(self.reverse_enum)):
      result += "    %s = %s;\n" % (v, k)

    result += "  }\n"

    result += self._FormatField()
    return result

Response: Return a string with the definition of this field.


Instruction:
Write a Python 3 function that can
check that value is a valid enum.

Response:
def Validate(self, value, **_):
    """Check that value is a valid enum."""
    if value is None:
      return

    return rdfvalue.RDFBool(super(ProtoBoolean, self).Validate(value))


Instruction:
How would you explain what the following Python 3 function does
def ConvertFromWireFormat(self, value, container=None):
    result = self.type()
    ReadIntoObject(value[2], 0, result)

    return result

Response: Converts a wire format string to a string.


Instruction:
Can you tell what is the following Python 3 function doing
def ConvertToWireFormat(self, value):
    output = _SerializeEntries(_GetOrderedEntries(value.GetRawData()))
    return (self.encoded_tag, VarintEncode(len(output)), output)

Response: Encode the nested protobuf into wire format.


Instruction:
How would you explain what the following Python 3 function does
def LateBind(self, target=None):
    if not issubclass(target, RDFProtoStruct):
      raise TypeError("Field %s expects a protobuf, but target is %s" %
                      (self, target))

    self.late_bound = False

    # The target type is now resolved.
    self.type = target

    # Register us in our owner.
    self.owner.AddDescriptor(self)

Response:
Late binding callback.

    This method is called on this field descriptor when the target RDFValue
    class is finally defined. It gives the field descriptor an opportunity to
    initialize after the point of definition.

    Args:
      target: The target nested class.

    Raises:
      TypeError: If the target class is not of the expected type.


Instruction:
Implement a function in Python 3 to
return and clear the dirty state of the python object.

Response:
def IsDirty(self, proto):
    """Return and clear the dirty state of the python object."""
    if proto.dirty:
      return True

    for python_format, _, type_descriptor in itervalues(proto.GetRawData()):
      if python_format is not None and type_descriptor.IsDirty(python_format):
        proto.dirty = True
        return True

    return False


Instruction:
Implement a function in Python 3 to
encode the nested protobuf into wire format.

Response:
def ConvertToWireFormat(self, value):
    """Encode the nested protobuf into wire format."""
    data = value.SerializeToString()
    return (self.encoded_tag, VarintEncode(len(data)), data)


Instruction:
Write a Python 3 script to
convert the wire format message into a value.

Response:
def ConvertFromWireFormat(self, value, container=None):
    """The wire format is an AnyValue message."""
    result = AnyValue()
    ReadIntoObject(value[2], 0, result)
    if self._type is not None:
      converted_value = self._type(container)
    else:
      converted_value = self._TypeFromAnyValue(result)

    # If one of the protobuf library wrapper classes is used, unwrap the value.
    if result.type_url.startswith("type.googleapis.com/google.protobuf."):
      wrapper_cls = self.__class__.WRAPPER_BY_TYPE[
          converted_value.data_store_type]
      wrapper_value = wrapper_cls()
      wrapper_value.ParseFromString(result.value)
      return converted_value.FromDatastoreValue(wrapper_value.value)
    else:
      # TODO(user): Type stored in type_url is currently ignored when value
      # is decoded. We should use it to deserialize the value and then check
      # that value type and dynamic type are compatible.
      return converted_value.FromSerializedString(result.value)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def ConvertToWireFormat(self, value):
    # Is it a protobuf-based value?
    if hasattr(value.__class__, "protobuf"):
      if value.__class__.protobuf:
        type_name = ("type.googleapis.com/%s" %
                     value.__class__.protobuf.__name__)
      else:
        type_name = value.__class__.__name__
      data = value.SerializeToString()
    # Is it a primitive value?
    elif hasattr(value.__class__, "data_store_type"):
      wrapper_cls = self.__class__.WRAPPER_BY_TYPE[
          value.__class__.data_store_type]
      wrapped_data = wrapper_cls()
      wrapped_data.value = value.SerializeToDataStore()

      type_name = ("type.googleapis.com/google.protobuf.%s" %
                   wrapper_cls.__name__)
      data = wrapped_data.SerializeToString()
    else:
      raise ValueError("Can't convert value %s to an protobuf.Any value." %
                       value)

    any_value = AnyValue(type_url=type_name, value=data)
    output = _SerializeEntries(_GetOrderedEntries(any_value.GetRawData()))

    return (self.encoded_tag, VarintEncode(len(output)), output)

Response: Encode the nested protobuf into wire format.


Instruction:
How would you explain what the following Python 3 function does
def Validate(self, value, **_):
    # Assigning from same kind can allow us to skip verification since all
    # elements in a RepeatedFieldHelper already are coerced to the delegate
    # type. In that case we just make a copy. This only works when the value
    # wraps the same type as us.
    if (value.__class__ is RepeatedFieldHelper and
        value.type_descriptor is self.delegate):
      result = value.Copy()

    # Make sure the base class finds the value valid.
    else:
      # The value may be a generator here, so we just iterate over it.
      result = RepeatedFieldHelper(type_descriptor=self.delegate)
      result.Extend(value)

    return result

Response: Check that value is a list of the required type.


Instruction:
Can you implement a function in Python 3 that
converts the value to the wire format.

Response:
def ConvertToWireFormat(self, value):
    """Convert to the wire format.

    Args:
      value: is of type RepeatedFieldHelper.

    Returns:
      A wire format representation of the value.
    """
    output = _SerializeEntries(
        (python_format, wire_format, value.type_descriptor)
        for (python_format, wire_format) in value.wrapped_list)
    return b"", b"", output


Instruction:
Can you write a function in Python 3 where it
adds a descriptor to the current descriptor.

Response:
def AddDescriptor(self, field_desc):
    """This method will be called by our delegate during late binding."""
    # Just relay it up to our owner.
    self.late_bound = False
    self.delegate = field_desc
    self.wire_type = self.delegate.wire_type
    self.owner.AddDescriptor(self)


Instruction:
Explain what the following Python 3 code does
def LateBind(self, target=None):
    self.type = target
    self._GetPrimitiveEncoder()

    # Now re-add the descriptor to the owner protobuf.
    self.late_bound = False
    self.owner.AddDescriptor(self)

Response: Bind the field descriptor to the owner once the target is defined.


Instruction:
Make a summary of the following Python 3 code
def _GetPrimitiveEncoder(self):
    # Decide what should the primitive type be for packing the target rdfvalue
    # into the protobuf and create a delegate descriptor to control that.
    primitive_cls = self._PROTO_DATA_STORE_LOOKUP[self.type.data_store_type]
    self.primitive_desc = primitive_cls(**self._kwargs)

    # Our wiretype is the same as the delegate's.
    self.wire_type = self.primitive_desc.wire_type
    self.proto_type_name = self.primitive_desc.proto_type_name

    # Recalculate our tags.
    self.CalculateTags()

Response: Finds the primitive encoder according to the type s data_store_type.


Instruction:
Here you have a function in Python 3, explain what it does
def Copy(self, field_number=None):
    new_args = self._kwargs.copy()
    if field_number is not None:
      new_args["field_number"] = field_number

    return ProtoRDFValue(
        rdf_type=self.original_proto_type_name,
        default=getattr(self, "default", None),
        **new_args)

Response: Returns a copy of this descriptor.


Instruction:
Can you generate the documentation for the following Python 3 function
def AsPrimitiveProto(self):
    if self.protobuf:
      result = self.protobuf()
      result.ParseFromString(self.SerializeToString())
      return result

Response: Return an old style protocol buffer object.


Instruction:
Write a Python 3 script for
initializing the internal structure from a given dictionary.

Response:
def FromDict(self, dictionary):
    """Initializes itself from a given dictionary."""
    dynamic_fields = []

    for key, value in iteritems(dictionary):
      field_type_info = self.type_infos.get(key)
      if isinstance(field_type_info, ProtoEmbedded):
        nested_value = field_type_info.GetDefault(container=self)
        nested_value.FromDict(value)
        self.Set(key, nested_value)
      elif isinstance(field_type_info, ProtoList):
        if isinstance(field_type_info.delegate, ProtoEmbedded):
          nested_values = []
          for v in value:
            nested_value = field_type_info.delegate.GetDefault(container=self)
            nested_value.FromDict(v)
            nested_values.append(nested_value)

          self.Set(key, nested_values)
        else:
          self.Set(key, value)
      elif isinstance(field_type_info, ProtoDynamicEmbedded):
        dynamic_fields.append(field_type_info)
      elif field_type_info.proto_type_name == "bytes":
        self.Set(key, base64.decodestring(value or ""))
      else:
        self.Set(key, value)

    # Process dynamic fields after all other fields, because most probably
    # their class is determined by one of the previously set fields.
    for dynamic_field in dynamic_fields:
      nested_value = dynamic_field.GetDefault(container=self)
      if nested_value is None:
        raise RuntimeError("Can't initialize dynamic field %s, probably some "
                           "necessary fields weren't supplied." %
                           dynamic_field.name)
      nested_value.FromDict(dictionary[dynamic_field.name])
      self.Set(dynamic_field.name, nested_value)


Instruction:
Write a Python 3 function that can
emit .proto file definitions.

Response:
def EmitProto(cls):
    """Emits .proto file definitions."""
    result = "message %s {\n" % cls.__name__
    for _, desc in sorted(iteritems(cls.type_infos_by_field_number)):
      result += desc.Definition()

    result += "}\n"
    return result


Instruction:
Explain what the following Python 3 code does
def FromTextFormat(cls, text):
    tmp = cls.protobuf()  # pylint: disable=not-callable
    text_format.Merge(text, tmp)

    return cls.FromSerializedString(tmp.SerializeToString())

Response: Parse this object from a text representation.


Instruction:
Can you tell what is the following Python 3 function doing
def AddDescriptor(cls, field_desc):
    if not isinstance(field_desc, ProtoType):
      raise type_info.TypeValueError(
          "%s field '%s' should be of type ProtoType" %
          (cls.__name__, field_desc.name))

    # Ensure the field descriptor knows the class that owns it.
    field_desc.SetOwner(cls)

    # If the field is late bound we do not really add it to the descriptor set
    # yet. We must wait for the LateBindingPlaceHolder() to add it later.
    if field_desc.late_bound:
      # Keep track of unbound fields.
      cls.late_bound_type_infos[field_desc.name] = field_desc
      return

    # Ensure this field number is unique:
    if field_desc.field_number in cls.type_infos_by_field_number:
      raise type_info.TypeValueError(
          "Field number %s for field %s is not unique in %s" %
          (field_desc.field_number, field_desc.name, cls.__name__))

    # We store an index of the type info by tag values to speed up parsing.
    cls.type_infos_by_field_number[field_desc.field_number] = field_desc
    cls.type_infos_by_encoded_tag[field_desc.encoded_tag] = field_desc

    cls.type_infos.Append(field_desc)
    cls.late_bound_type_infos.pop(field_desc.name, None)

    # Add direct accessors only if the class does not already have them.
    if not hasattr(cls, field_desc.name):
      # This lambda is a class method so pylint: disable=protected-access
      # This is much faster than __setattr__/__getattr__
      setattr(
          cls, field_desc.name,
          property(lambda self: self.Get(field_desc.name), lambda self, x: self.
                   _Set(x, field_desc), None, field_desc.description))

Response: Adds a descriptor to the Proto Struct.


Instruction:
Explain what the following Python 3 code does
def RunInstaller():

  try:
    os.makedirs(os.path.dirname(config.CONFIG["Installer.logfile"]))
  except OSError:
    pass

  # Always log to the installer logfile at debug level. This way if our
  # installer fails we can send detailed diagnostics.
  handler = logging.FileHandler(config.CONFIG["Installer.logfile"], mode="wb")

  handler.setLevel(logging.DEBUG)

  # Add this to the root logger.
  logging.getLogger().addHandler(handler)

  # Ordinarily when the client starts up, the local volatile
  # configuration is read. Howevwer, when running the installer, we
  # need to ensure that only the installer configuration is used so
  # nothing gets overridden by local settings. We there must reload
  # the configuration from the flag and ignore the Config.writeback
  # location.
  config.CONFIG.Initialize(filename=flags.FLAGS.config, reset=True)
  config.CONFIG.AddContext(contexts.INSTALLER_CONTEXT,
                           "Context applied when we run the client installer.")

  logging.warning("Starting installation procedure for GRR client.")
  try:
    Installer().Init()
  except Exception as e:  # pylint: disable=broad-except
    # Ouch! we failed to install... Not a lot we can do
    # here - just log the error and give up.
    logging.exception("Installation failed: %s", e)

    # Error return status.
    sys.exit(-1)

  # Exit successfully.
  sys.exit(0)

Response: Run all the current installers and exit the process.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def MapItemsIterator(function, items):
  return ItemsIterator(
      items=map(function, items), total_count=items.total_count)

Response: Maps ItemsIterator via given function.


Instruction:
Make a summary of the following Python 3 code
def Poll(generator=None, condition=None, interval=None, timeout=None):

  if not generator:
    raise ValueError("generator has to be a lambda")

  if not condition:
    raise ValueError("condition has to be a lambda")

  if interval is None:
    interval = DEFAULT_POLL_INTERVAL

  if timeout is None:
    timeout = DEFAULT_POLL_TIMEOUT

  started = time.time()
  while True:
    obj = generator()
    check_result = condition(obj)
    if check_result:
      return obj

    if timeout and (time.time() - started) > timeout:
      raise errors.PollTimeoutError(
          "Polling on %s timed out after %ds." % (obj, timeout))
    time.sleep(interval)

Response: Periodically calls generator function until a condition is satisfied.


Instruction:
Can you implement a function in Python 3 that
converts given URN string to a client id string.

Response:
def UrnStringToClientId(urn):
  """Converts given URN string to a client id string."""
  if urn.startswith(AFF4_PREFIX):
    urn = urn[len(AFF4_PREFIX):]

  components = urn.split("/")
  return components[0]


Instruction:
Given the following Python 3 function, write the documentation
def UrnStringToHuntId(urn):
  if urn.startswith(AFF4_PREFIX):
    urn = urn[len(AFF4_PREFIX):]

  components = urn.split("/")
  if len(components) != 2 or components[0] != "hunts":
    raise ValueError("Invalid hunt URN: %s" % urn)

  return components[-1]

Response: Converts given URN string to a flow id string.


Instruction:
Can you create a Python 3 function that
returns a message instance corresponding to a given type URL.

Response:
def TypeUrlToMessage(type_url):
  """Returns a message instance corresponding to a given type URL."""

  if not type_url.startswith(TYPE_URL_PREFIX):
    raise ValueError("Type URL has to start with a prefix %s: %s" %
                     (TYPE_URL_PREFIX, type_url))

  full_name = type_url[len(TYPE_URL_PREFIX):]
  try:
    return symbol_database.Default().GetSymbol(full_name)()
  except KeyError as e:
    raise ProtobufTypeNotFound(str(e))


Instruction:
Write a Python 3 function that can
register all API - releated descriptors in a given symbol DB.

Response:
def RegisterProtoDescriptors(db, *additional_descriptors):
  """Registers all API-releated descriptors in a given symbol DB."""
  db.RegisterFileDescriptor(artifact_pb2.DESCRIPTOR)
  db.RegisterFileDescriptor(client_pb2.DESCRIPTOR)
  db.RegisterFileDescriptor(config_pb2.DESCRIPTOR)
  db.RegisterFileDescriptor(cron_pb2.DESCRIPTOR)
  db.RegisterFileDescriptor(flow_pb2.DESCRIPTOR)
  db.RegisterFileDescriptor(hunt_pb2.DESCRIPTOR)
  db.RegisterFileDescriptor(output_plugin_pb2.DESCRIPTOR)
  db.RegisterFileDescriptor(reflection_pb2.DESCRIPTOR)
  db.RegisterFileDescriptor(stats_pb2.DESCRIPTOR)
  db.RegisterFileDescriptor(user_pb2.DESCRIPTOR)
  db.RegisterFileDescriptor(vfs_pb2.DESCRIPTOR)

  db.RegisterFileDescriptor(checks_pb2.DESCRIPTOR)
  db.RegisterFileDescriptor(deprecated_pb2.DESCRIPTOR)
  db.RegisterFileDescriptor(flows_pb2.DESCRIPTOR)
  db.RegisterFileDescriptor(jobs_pb2.DESCRIPTOR)
  db.RegisterFileDescriptor(osquery_pb2.DESCRIPTOR)
  db.RegisterFileDescriptor(wrappers_pb2.DESCRIPTOR)

  for d in additional_descriptors:
    db.RegisterFileDescriptor(d)


Instruction:
Can you create a Python 3 function that
converts python datetime to epoch microseconds.

Response:
def _DateToEpoch(date):
  """Converts python datetime to epoch microseconds."""
  tz_zero = datetime.datetime.utcfromtimestamp(0)
  diff_sec = int((date - tz_zero).total_seconds())
  return diff_sec * 1000000


Instruction:
Implement a Python 3 function for
parsing the stat entries.

Response:
def ParseMultiple(self, stat_entries, knowledge_base):
    """Parse the StatEntry objects."""
    _ = knowledge_base

    for stat_entry in stat_entries:
      # TODO: `st_mode` has to be an `int`, not `StatMode`.
      if stat.S_ISDIR(int(stat_entry.st_mode)):
        homedir = stat_entry.pathspec.path
        username = os.path.basename(homedir)
        if username not in self.blacklist:
          yield rdf_client.User(username=username, homedir=homedir)


Instruction:
Can you implement a function in Python 3 that
parses the system profiler output.

Response:
def Parse(self, cmd, args, stdout, stderr, return_val, time_taken,
            knowledge_base):
    """Parse the system profiler output. We get it in the form of a plist."""
    _ = stderr, time_taken, args, knowledge_base  # Unused
    self.CheckReturn(cmd, return_val)

    plist = biplist.readPlist(io.BytesIO(stdout))

    if len(plist) > 1:
      raise parser.ParseError("SPHardwareDataType plist has too many items.")

    hardware_list = plist[0]["_items"][0]
    serial_number = hardware_list.get("serial_number", None)
    system_product_name = hardware_list.get("machine_model", None)
    bios_version = hardware_list.get("boot_rom_version", None)

    yield rdf_client.HardwareInfo(
        serial_number=serial_number,
        bios_version=bios_version,
        system_product_name=system_product_name)


Instruction:
Given the following Python 3 function, write the documentation
def Parse(self, statentry, file_object, knowledge_base):
    _ = knowledge_base
    kwargs = {}
    try:
      kwargs["aff4path"] = file_object.urn
    except AttributeError:
      pass

    direct_copy_items = [
        "Label", "Disabled", "UserName", "GroupName", "Program",
        "StandardInPath", "StandardOutPath", "StandardErrorPath",
        "LimitLoadToSessionType", "EnableGlobbing", "EnableTransactions",
        "OnDemand", "RunAtLoad", "RootDirectory", "WorkingDirectory", "Umask",
        "TimeOut", "ExitTimeOut", "ThrottleInterval", "InitGroups",
        "StartOnMount", "StartInterval", "Debug", "WaitForDebugger", "Nice",
        "ProcessType", "AbandonProcessGroup", "LowPriorityIO", "LaunchOnlyOnce"
    ]

    string_array_items = [
        "LimitLoadToHosts", "LimitLoadFromHosts", "LimitLoadToSessionType",
        "ProgramArguments", "WatchPaths", "QueueDirectories"
    ]

    flag_only_items = ["SoftResourceLimits", "HardResourceLimits", "Sockets"]

    plist = {}

    try:
      plist = biplist.readPlist(file_object)
    except (biplist.InvalidPlistException, ValueError, IOError) as e:
      plist["Label"] = "Could not parse plist: %s" % e

    # These are items that can be directly copied
    for key in direct_copy_items:
      kwargs[key] = plist.get(key)

    # These could be a string, they could be an array, we don't know and neither
    # does Apple so we check.
    for key in string_array_items:
      elements = plist.get(key)
      if isinstance(elements, string_types):
        kwargs[key] = [elements]
      else:
        kwargs[key] = elements

    # These are complex items that can appear in multiple data structures
    # so we only flag on their existence
    for key in flag_only_items:
      if plist.get(key):
        kwargs[key] = True

    if plist.get("inetdCompatability") is not None:
      kwargs["inetdCompatabilityWait"] = plist.get("inetdCompatability").get(
          "Wait")

    keepalive = plist.get("KeepAlive")
    if isinstance(keepalive, bool) or keepalive is None:
      kwargs["KeepAlive"] = keepalive
    else:
      keepalivedict = {}
      keepalivedict["SuccessfulExit"] = keepalive.get("SuccessfulExit")
      keepalivedict["NetworkState"] = keepalive.get("NetworkState")

      pathstates = keepalive.get("PathState")
      if pathstates is not None:
        keepalivedict["PathState"] = []
        for pathstate in pathstates:
          keepalivedict["PathState"].append(
              rdf_plist.PlistBoolDictEntry(
                  name=pathstate, value=pathstates[pathstate]))

      otherjobs = keepalive.get("OtherJobEnabled")
      if otherjobs is not None:
        keepalivedict["OtherJobEnabled"] = []
        for otherjob in otherjobs:
          keepalivedict["OtherJobEnabled"].append(
              rdf_plist.PlistBoolDictEntry(
                  name=otherjob, value=otherjobs[otherjob]))
      kwargs["KeepAliveDict"] = rdf_plist.LaunchdKeepAlive(**keepalivedict)

    envvars = plist.get("EnvironmentVariables")
    if envvars is not None:
      kwargs["EnvironmentVariables"] = []
      for envvar in envvars:
        kwargs["EnvironmentVariables"].append(
            rdf_plist.PlistStringDictEntry(name=envvar, value=envvars[envvar]))

    startcalendarinterval = plist.get("StartCalendarInterval")
    if startcalendarinterval is not None:
      if isinstance(startcalendarinterval, dict):
        kwargs["StartCalendarInterval"] = [
            rdf_plist.LaunchdStartCalendarIntervalEntry(
                Minute=startcalendarinterval.get("Minute"),
                Hour=startcalendarinterval.get("Hour"),
                Day=startcalendarinterval.get("Day"),
                Weekday=startcalendarinterval.get("Weekday"),
                Month=startcalendarinterval.get("Month"))
        ]
      else:
        kwargs["StartCalendarInterval"] = []
        for entry in startcalendarinterval:
          kwargs["StartCalendarInterval"].append(
              rdf_plist.LaunchdStartCalendarIntervalEntry(
                  Minute=entry.get("Minute"),
                  Hour=entry.get("Hour"),
                  Day=entry.get("Day"),
                  Weekday=entry.get("Weekday"),
                  Month=entry.get("Month")))

    yield rdf_plist.LaunchdPlist(**kwargs)

Response: Parse the Plist file.


Instruction:
Create a Python 3 function for
parsing the install history plist file.

Response:
def Parse(self, statentry, file_object, knowledge_base):
    """Parse the Plist file."""

    plist = biplist.readPlist(file_object)

    if not isinstance(plist, list):
      raise parser.ParseError(
          "InstallHistory plist is a '%s', expecting a list" % type(plist))

    packages = []
    for sw in plist:
      packages.append(
          rdf_client.SoftwarePackage(
              name=sw.get("displayName"),
              version=sw.get("displayVersion"),
              description=",".join(sw.get("packageIdentifiers")),
              # TODO(hanuszczak): make installed_on an RDFDatetime
              installed_on=_DateToEpoch(sw.get("date")),
              install_state=rdf_client.SoftwarePackage.InstallState.INSTALLED))

    if packages:
      yield rdf_client.SoftwarePackages(packages=packages)


Instruction:
Can you write a function in Python 3 where it
adds the running contexts to the config system.

Response:
def SetPlatformArchContext():
  """Add the running contexts to the config system."""

  # Initialize the running platform context:
  _CONFIG.AddContext("Platform:%s" % platform.system().title())

  machine = platform.uname()[4]
  if machine in ["x86_64", "AMD64", "i686"]:
    # 32 bit binaries running on AMD64 will still have a i386 arch.
    if platform.architecture()[0] == "32bit":
      arch = "i386"
    else:
      arch = "amd64"
  elif machine == "x86":
    arch = "i386"
  else:
    arch = machine

  _CONFIG.AddContext("Arch:%s" % arch)


Instruction:
Implement a Python 3 function for
parsing given YAML file.

Response:
def _ParseYamlFromFile(filedesc):
  """Parses given YAML file."""
  content = filedesc.read()
  return yaml.Parse(content) or collections.OrderedDict()


Instruction:
Given the following Python 3 function, write the documentation
def DEFINE_choice(name, default, choices, help):
  _CONFIG.DEFINE_choice(name, default, choices, help)

Response: A helper for defining choice string options.


Instruction:
Can you tell what is the following Python 3 function doing
def DEFINE_multichoice(name, default, choices, help):
  _CONFIG.DEFINE_multichoice(name, default, choices, help)

Response: Defines a new option for a list of choices.


Instruction:
Create a Python 3 function to
initialize a ConfigManager object with the specified options.

Response:
def LoadConfig(config_obj,
               config_file=None,
               config_fd=None,
               secondary_configs=None,
               contexts=None,
               reset=False,
               parser=ConfigFileParser):
  """Initialize a ConfigManager with the specified options.

  Args:
    config_obj: The ConfigManager object to use and update. If None, one will be
      created.
    config_file: Filename to read the config from.
    config_fd: A file-like object to read config data from.
    secondary_configs: A list of secondary config URLs to load.
    contexts: Add these contexts to the config object.
    reset: Completely wipe previous config before doing the load.
    parser: Specify which parser to use.

  Returns:
    The resulting config object. The one passed in, unless None was specified.
  """
  if config_obj is None or reset:
    # Create a new config object.
    config_obj = _CONFIG.MakeNewConfig()

  # Initialize the config with a filename or file like object.
  if config_file is not None:
    config_obj.Initialize(filename=config_file, must_exist=True, parser=parser)
  elif config_fd is not None:
    config_obj.Initialize(fd=config_fd, parser=parser)

  # Load all secondary files.
  if secondary_configs:
    for config_file in secondary_configs:
      config_obj.LoadSecondaryConfig(config_file)

  if contexts:
    for context in contexts:
      config_obj.AddContext(context)

  return config_obj