Instruction:
Make a summary of the following Python 3 code
def GetValue(self, ignore_error=True):
    if self.HasField("none"):
      return None

    field_names = [
        "integer", "string", "data", "boolean", "list", "dict", "rdf_value",
        "float", "set"
    ]

    values = [getattr(self, x) for x in field_names if self.HasField(x)]

    if len(values) != 1:
      return None

    if self.HasField("boolean"):
      return bool(values[0])

    # Unpack RDFValues.
    if self.HasField("rdf_value"):
      try:
        rdf_class = rdfvalue.RDFValue.classes[self.rdf_value.name]
        return rdf_class.FromSerializedString(
            self.rdf_value.data, age=self.rdf_value.age)
      except (ValueError, KeyError) as e:
        if ignore_error:
          return e

        raise

    elif self.HasField("list"):
      return [x.GetValue() for x in self.list.content]

    elif self.HasField("set"):
      return set([x.GetValue() for x in self.set.content])

    else:
      return values[0]

Response: Extracts and returns a single value from a DataBlob.


Instruction:
Can you tell what is the following Python 3 function doing
def Append(self, value=None, **kwarg):
    if self.rdf_type is not None:
      if (isinstance(value, rdfvalue.RDFValue) and
          value.__class__ != self.rdf_type):
        raise ValueError("Can only accept %s" % self.rdf_type)

      try:
        # Try to coerce the value.
        value = self.rdf_type(value, **kwarg)  # pylint: disable=not-callable
      except (TypeError, ValueError):
        raise ValueError("Unable to initialize %s from type %s" %
                         (self.__class__.__name__, type(value)))

    self.content.Append(DataBlob().SetValue(value))

Response: Append a new element to the array.


Instruction:
Given the following Python 3 function, write the documentation
def FromMany(cls, samples):
    if not samples:
      raise ValueError("Empty `samples` argument")

    # It only makes sense to average the CPU percentage. For all other values
    # we simply take the biggest of them.
    cpu_percent = sum(sample.cpu_percent for sample in samples) / len(samples)

    return CpuSample(
        timestamp=max(sample.timestamp for sample in samples),
        cpu_percent=cpu_percent,
        user_cpu_time=max(sample.user_cpu_time for sample in samples),
        system_cpu_time=max(sample.system_cpu_time for sample in samples))

Response: Constructs a single sample that best represents a list of samples.


Instruction:
How would you explain what the following Python 3 function does
def FromMany(cls, samples):
    if not samples:
      raise ValueError("Empty `samples` argument")

    return IOSample(
        timestamp=max(sample.timestamp for sample in samples),
        read_bytes=max(sample.read_bytes for sample in samples),
        write_bytes=max(sample.write_bytes for sample in samples))

Response: Constructs a single sample that best represents a list of samples.


Instruction:
Can you write a function in Python 3 where it
constructs a copy of given stats but downsampled to given interval.

Response:
def Downsampled(cls, stats, interval=None):
    """Constructs a copy of given stats but downsampled to given interval.

    Args:
      stats: A `ClientStats` instance.
      interval: A downsampling interval.

    Returns:
      A downsampled `ClientStats` instance.
    """
    interval = interval or cls.DEFAULT_SAMPLING_INTERVAL

    result = cls(stats)
    result.cpu_samples = cls._Downsample(
        kind=CpuSample, samples=stats.cpu_samples, interval=interval)
    result.io_samples = cls._Downsample(
        kind=IOSample, samples=stats.io_samples, interval=interval)
    return result


Instruction:
Given the following Python 3 function, write the documentation
def GetUserInfo(knowledge_base, user):
  # TODO: This docstring cannot be a raw literal because there are
  # issues with raw unicode literals on Python 2. Once support for Python 2 is
  # dropped, it can be made raw again.
  # pylint: disable=g-docstring-has-escape
  """Get a User protobuf for a specific user.

  Args:
    knowledge_base: An rdf_client.KnowledgeBase object.
    user: Username as string. May contain domain like DOMAIN\\user.

  Returns:
    A User rdfvalue or None
  """
  # pylint: enable=g-docstring-has-escape
  if "\\" in user:
    domain, user = user.split("\\", 1)
    users = [
        u for u in knowledge_base.users
        if u.username == user and u.userdomain == domain
    ]
  else:
    users = [u for u in knowledge_base.users if u.username == user]

  if not users:
    return
  else:
    return users[0]

Response: Get a User protobuf for a specific user.


Instruction:
Here you have a function in Python 3, explain what it does
def UpdateVFSFileAndWait(client_id,
                         vfs_file_urn,
                         token=None,
                         timeout=DEFAULT_TIMEOUT):
  """Waits for a file to be updated on the client.

  Calls the UpdateVFSFile flow on a urn and waits for both it and the
  ListDirectory flow it calls to finish.

  Note that this is needed because any flows UpdateVFSFile calls via
  VFS Update methods will not become child flows of UpdateVFSFile,
  and therefore waiting for UpdateVFSFile to complete is not enough.

  Args:
    client_id: Which client to run the flow on.
    vfs_file_urn: Path to VFSFile to update.
    token: The datastore access token.
    timeout: How long to wait for a flow to finish, maximum.
  """
  # Wait for the UpdateVFSFile flow.
  update_flow_urn = StartFlowAndWait(
      client_id,
      token=token,
      timeout=timeout,
      flow_name=aff4_grr.UpdateVFSFile.__name__,
      vfs_file_urn=vfs_file_urn)

  update_flow_obj = aff4.FACTORY.Open(
      update_flow_urn, token=token, aff4_type=flow.GRRFlow)

  # Get the child flow so we can wait for it too.
  sub_flow_urn = update_flow_obj.state.get_file_flow_urn

  # If there was no subflow, no need to wait for it.
  if not sub_flow_urn:
    return

  WaitForFlow(sub_flow_urn, token=token, timeout=timeout)

Response: Calls the UpdateVFSFile flow and waits for it to finish.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def WaitForFlow(flow_urn,
                token=None,
                timeout=DEFAULT_TIMEOUT,
                max_sleep_time=1,
                min_sleep_time=0.2,
                dampening_multiplier=0.9):
  """Waits for a flow to finish, polling while we wait.

  Args:
    flow_urn: The urn of the flow to wait for.
    token: The datastore access token.
    timeout: How long to wait before giving up, usually because the client has
      gone away.
    max_sleep_time: The initial and longest time to wait in between polls.
    min_sleep_time: The final and shortest time to wait in between polls.
    dampening_multiplier: The current sleep time is multiplied by this number on
      each iteration. Controls how fast the polling reaches its minimum sleep
      time. You probably want this to be less than 1, unless you want to wait an
      increasing amount of time in between flows.

  Raises:
    IOError: If we time out while waiting for the client.
  """

  start_time = time.time()
  sleep_time = max_sleep_time
  while True:
    # Reopen the AFF4Object to check if its status has changed, and also make
    # sure it's a flow.
    with aff4.FACTORY.Open(
        flow_urn, token=token, aff4_type=flow.GRRFlow) as flow_obj:

      # Stop if the flow is done or has timed out.
      if time.time() - start_time > timeout:
        logging.warning("Timed out after waiting %ss for %s!", timeout,
                        flow_obj)
        raise IOError("Timed out trying to access client! Is it connected?")
      if not flow_obj.GetRunner().IsRunning():
        break
    # Decrease the time we sleep each iteration.
    sleep_time = max(sleep_time * dampening_multiplier, min_sleep_time)
    time.sleep(sleep_time)
    logging.debug("Waiting for %s, sleeping for %.3fs", flow_obj, sleep_time)

Response: Waits for a flow to finish and returns it.


Instruction:
Create a Python 3 function to
run a flow and waits for it to finish.

Response:
def StartFlowAndWait(client_id,
                     token=None,
                     timeout=DEFAULT_TIMEOUT,
                     **flow_args):
  """Runs a flow and waits for it to finish.

  Args:
    client_id: The client id of the client to run on.
    token: The datastore access token.
    timeout: How long to wait for a flow to complete, maximum.
    **flow_args: Pass through to flow.

  Returns:
    The urn of the flow that was run.
  """
  flow_urn = flow.StartAFF4Flow(
      client_id=client_id, token=token, sync=True, **flow_args)

  WaitForFlow(flow_urn, token=token, timeout=timeout)

  return flow_urn


Instruction:
How would you explain what the following Python 3 function does
def InterpolatePath(path, knowledge_base, users=None, path_args=None, depth=0):

  sys_formatters = {
      # TODO(user): Collect this during discovery from the registry.
      # HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows NT\CurrentVersion\
      # Value: SystemRoot
      "systemroot": "c:\\Windows"
  }

  # Override any system formatters with path_args.
  if path_args:
    sys_formatters.update(path_args)

  if users:
    results = []
    for user in users:
      # Extract and interpolate user specific formatters.
      user = GetUserInfo(knowledge_base, user)
      if user:
        formatters = dict((x.name, y) for x, y in user.ListSetFields())
        formatters.update(sys_formatters)
        try:
          results.append(path.format(**formatters))
        except KeyError:
          pass  # We may be missing values for some users.
    return results
  else:
    try:
      path = path.format(**sys_formatters)
    except KeyError:
      logging.warning("Failed path interpolation on %s", path)
      return ""
    if "{" in path and depth < 10:
      path = InterpolatePath(
          path,
          knowledge_base=knowledge_base,
          users=users,
          path_args=path_args,
          depth=depth + 1)
    return path

Response: Interpolate a path on a client and return a single string.


Instruction:
Create a Python 3 function for
building a list of filter callables.

Response:
def BuildChecks(self, request):
    """Parses request and returns a list of filter callables.

    Each callable will be called with the StatEntry and returns True if the
    entry should be suppressed.

    Args:
      request: A FindSpec that describes the search.

    Returns:
      a list of callables which return True if the file is to be suppressed.
    """
    result = []
    if request.HasField("start_time") or request.HasField("end_time"):

      def FilterTimestamp(file_stat, request=request):
        return file_stat.HasField("st_mtime") and (
            file_stat.st_mtime < request.start_time or
            file_stat.st_mtime > request.end_time)

      result.append(FilterTimestamp)

    if request.HasField("min_file_size") or request.HasField("max_file_size"):

      def FilterSize(file_stat, request=request):
        return file_stat.HasField("st_size") and (
            file_stat.st_size < request.min_file_size or
            file_stat.st_size > request.max_file_size)

      result.append(FilterSize)

    if request.HasField("perm_mode"):

      def FilterPerms(file_stat, request=request):
        return (file_stat.st_mode & request.perm_mask) != request.perm_mode

      result.append(FilterPerms)

    if request.HasField("uid"):

      def FilterUID(file_stat, request=request):
        return file_stat.st_uid != request.uid

      result.append(FilterUID)

    if request.HasField("gid"):

      def FilterGID(file_stat, request=request):
        return file_stat.st_gid != request.gid

      result.append(FilterGID)

    if request.HasField("path_regex"):
      regex = request.path_regex

      def FilterPath(file_stat, regex=regex):
        """Suppress any filename not matching the regular expression."""
        return not regex.Search(file_stat.pathspec.Basename())

      result.append(FilterPath)

    if request.HasField("data_regex"):

      def FilterData(file_stat, **_):
        """Suppress files that do not match the content."""
        return not self.TestFileContent(file_stat)

      result.append(FilterData)

    return result


Instruction:
How would you code a function in Python 3 to
run the Find action.

Response:
def Run(self, request):
    """Runs the Find action."""

    self.request = request
    filters = self.BuildChecks(request)

    files_checked = 0
    for f in self.ListDirectory(request.pathspec):
      self.Progress()

      # Ignore this file if any of the checks fail.
      if not any((check(f) for check in filters)):
        self.SendReply(f)

      files_checked += 1
      if files_checked >= self.MAX_FILES_TO_CHECK:
        return


Instruction:
Can you implement a function in Python 3 that
searches the data for a hit in the given regex.

Response:
def FindRegex(self, regex, data):
    """Search the data for a hit."""
    for match in re.finditer(regex, data, flags=re.I | re.S | re.M):
      yield (match.start(), match.end())


Instruction:
Can you create a Python 3 function that
searches the data for a hit.

Response:
def FindLiteral(self, pattern, data):
    """Search the data for a hit."""
    pattern = utils.Xor(pattern, self.xor_in_key)

    offset = 0
    while 1:
      # We assume here that data.find does not make a copy of pattern.
      offset = data.find(pattern, offset)

      if offset < 0:
        break

      yield (offset, offset + len(pattern))

      offset += 1


Instruction:
Here you have a function in Python 3, explain what it does
def Run(self, args):
    fd = vfs.VFSOpen(args.target, progress_callback=self.Progress)
    fd.Seek(args.start_offset)
    base_offset = args.start_offset

    self.xor_in_key = args.xor_in_key
    self.xor_out_key = args.xor_out_key

    if args.regex:
      find_func = functools.partial(self.FindRegex, args.regex.AsBytes())
    elif args.literal:
      find_func = functools.partial(self.FindLiteral, args.literal.AsBytes())
    else:
      raise RuntimeError("Grep needs a regex or a literal.")

    preamble_size = 0
    postscript_size = 0
    hits = 0
    data = b""
    while fd.Tell() < args.start_offset + args.length:

      # Base size to read is at most the buffer size.
      to_read = min(args.length, self.BUFF_SIZE,
                    args.start_offset + args.length - fd.Tell())
      # Read some more data for the snippet.
      to_read += self.ENVELOPE_SIZE - postscript_size
      read_data = fd.Read(to_read)

      data = data[-postscript_size - self.ENVELOPE_SIZE:] + read_data

      postscript_size = max(0, self.ENVELOPE_SIZE - (to_read - len(read_data)))
      data_size = len(data) - preamble_size - postscript_size

      if data_size == 0 and postscript_size == 0:
        break

      for (start, end) in find_func(data):
        # Ignore hits in the preamble.
        if end <= preamble_size:
          continue

        # Ignore hits in the postscript.
        if end > preamble_size + data_size:
          continue

        # Offset of file in the end after length.
        if end + base_offset - preamble_size > args.start_offset + args.length:
          break

        data_start = max(0, start - args.bytes_before)
        data_end = min(len(data), end + args.bytes_after)
        out_data = utils.Xor(data[data_start:data_end], self.xor_out_key)

        hits += 1
        self.SendReply(
            rdf_client.BufferReference(
                offset=base_offset + start - preamble_size,
                data=out_data,
                length=len(out_data),
                pathspec=fd.pathspec))

        if args.mode == rdf_client_fs.GrepSpec.Mode.FIRST_HIT:
          return

        if hits >= self.HIT_LIMIT:
          msg = utils.Xor(
              b"This Grep has reached the maximum number of hits"
              b" (%d)." % self.HIT_LIMIT, self.xor_out_key)
          self.SendReply(
              rdf_client.BufferReference(offset=0, data=msg, length=len(msg)))
          return

      self.Progress()

      base_offset += data_size

      # Allow for overlap with previous matches.
      preamble_size = min(len(data), self.ENVELOPE_SIZE)

Response: This function searches the file for a pattern and returns the data.


Instruction:
How would you implement a function in Python 3 that
stops the hunt if number of crashes exceeds the limit.

Response:
def StopHuntIfCrashLimitExceeded(hunt_id):
  """Stops the hunt if number of crashes exceeds the limit."""
  hunt_obj = data_store.REL_DB.ReadHuntObject(hunt_id)

  # Do nothing if the hunt is already stopped.
  if hunt_obj.hunt_state == rdf_hunt_objects.Hunt.HuntState.STOPPED:
    return hunt_obj

  if hunt_obj.crash_limit:
    hunt_counters = data_store.REL_DB.ReadHuntCounters(hunt_id)
    if hunt_counters.num_crashed_clients >= hunt_obj.crash_limit:
      # Remove our rules from the forman and cancel all the started flows.
      # Hunt will be hard-stopped and it will be impossible to restart it.
      reason = ("Hunt %s reached the crashes limit of %d "
                "and was stopped.") % (hunt_obj.hunt_id, hunt_obj.crash_limit)
      StopHunt(hunt_obj.hunt_id, reason=reason)

  return hunt_obj


Instruction:
Write a Python 3 script for
stopping the hunt if CPU or network limits are exceeded.

Response:
def StopHuntIfCPUOrNetworkLimitsExceeded(hunt_id):
  """Stops the hunt if average limites are exceeded."""
  hunt_obj = data_store.REL_DB.ReadHuntObject(hunt_id)

  # Do nothing if the hunt is already stopped.
  if hunt_obj.hunt_state == rdf_hunt_objects.Hunt.HuntState.STOPPED:
    return hunt_obj

  hunt_counters = data_store.REL_DB.ReadHuntCounters(hunt_id)

  # Check global hunt network bytes limit first.
  if (hunt_obj.total_network_bytes_limit and
      hunt_counters.total_network_bytes_sent >
      hunt_obj.total_network_bytes_limit):
    reason = ("Hunt %s reached the total network bytes sent limit of %d and "
              "was stopped.") % (hunt_obj.hunt_id,
                                 hunt_obj.total_network_bytes_limit)
    return StopHunt(hunt_obj.hunt_id, reason=reason)

  # Check that we have enough clients to apply average limits.
  if hunt_counters.num_clients < MIN_CLIENTS_FOR_AVERAGE_THRESHOLDS:
    return hunt_obj

  # Check average per-client results count limit.
  if hunt_obj.avg_results_per_client_limit:
    avg_results_per_client = (
        hunt_counters.num_results / hunt_counters.num_clients)
    if avg_results_per_client > hunt_obj.avg_results_per_client_limit:
      # Stop the hunt since we get too many results per client.
      reason = ("Hunt %s reached the average results per client "
                "limit of %d and was stopped.") % (
                    hunt_obj.hunt_id, hunt_obj.avg_results_per_client_limit)
      return StopHunt(hunt_obj.hunt_id, reason=reason)

  # Check average per-client CPU seconds limit.
  if hunt_obj.avg_cpu_seconds_per_client_limit:
    avg_cpu_seconds_per_client = (
        hunt_counters.total_cpu_seconds / hunt_counters.num_clients)
    if avg_cpu_seconds_per_client > hunt_obj.avg_cpu_seconds_per_client_limit:
      # Stop the hunt since we use too many CPUs per client.
      reason = ("Hunt %s reached the average CPU seconds per client "
                "limit of %d and was stopped.") % (
                    hunt_obj.hunt_id, hunt_obj.avg_cpu_seconds_per_client_limit)
      return StopHunt(hunt_obj.hunt_id, reason=reason)

  # Check average per-client network bytes limit.
  if hunt_obj.avg_network_bytes_per_client_limit:
    avg_network_bytes_per_client = (
        hunt_counters.total_network_bytes_sent / hunt_counters.num_clients)
    if (avg_network_bytes_per_client >
        hunt_obj.avg_network_bytes_per_client_limit):
      # Stop the hunt since we use too many network bytes sent
      # per client.
      reason = ("Hunt %s reached the average network bytes per client "
                "limit of %d and was stopped.") % (
                    hunt_obj.hunt_id,
                    hunt_obj.avg_network_bytes_per_client_limit)
      return StopHunt(hunt_obj.hunt_id, reason=reason)

  return hunt_obj


Instruction:
How would you explain what the following Python 3 function does
def CompleteHuntIfExpirationTimeReached(hunt_obj):
  # TODO(hanuszczak): This should not set the hunt state to `COMPLETED` but we
  # should have a sparate `EXPIRED` state instead and set that.
  if (hunt_obj.hunt_state not in [
      rdf_hunt_objects.Hunt.HuntState.STOPPED,
      rdf_hunt_objects.Hunt.HuntState.COMPLETED
  ] and hunt_obj.expired):
    StopHunt(hunt_obj.hunt_id, reason="Hunt completed.")

    data_store.REL_DB.UpdateHuntObject(
        hunt_obj.hunt_id, hunt_state=hunt_obj.HuntState.COMPLETED)
    return data_store.REL_DB.ReadHuntObject(hunt_obj.hunt_id)

  return hunt_obj

Response: Marks the hunt as complete if it s past its expiry time.


Instruction:
Can you implement a function in Python 3 that
creates a hunt using a given hunt object.

Response:
def CreateHunt(hunt_obj):
  """Creates a hunt using a given hunt object."""
  data_store.REL_DB.WriteHuntObject(hunt_obj)

  if hunt_obj.HasField("output_plugins"):
    output_plugins_states = flow.GetOutputPluginStates(
        hunt_obj.output_plugins,
        source="hunts/%s" % hunt_obj.hunt_id,
        token=access_control.ACLToken(username=hunt_obj.creator))
    data_store.REL_DB.WriteHuntOutputPluginsStates(hunt_obj.hunt_id,
                                                   output_plugins_states)


Instruction:
Here you have a function in Python 3, explain what it does
def CreateAndStartHunt(flow_name, flow_args, creator, **kwargs):

  # This interface takes a time when the hunt expires. However, the legacy hunt
  # starting interface took an rdfvalue.Duration object which was then added to
  # the current time to get the expiry. This check exists to make sure we don't
  # confuse the two.
  if "duration" in kwargs:
    precondition.AssertType(kwargs["duration"], rdfvalue.Duration)

  hunt_args = rdf_hunt_objects.HuntArguments(
      hunt_type=rdf_hunt_objects.HuntArguments.HuntType.STANDARD,
      standard=rdf_hunt_objects.HuntArgumentsStandard(
          flow_name=flow_name, flow_args=flow_args))

  hunt_obj = rdf_hunt_objects.Hunt(
      creator=creator,
      args=hunt_args,
      create_time=rdfvalue.RDFDatetime.Now(),
      **kwargs)

  CreateHunt(hunt_obj)
  StartHunt(hunt_obj.hunt_id)

  return hunt_obj.hunt_id

Response: Creates and starts a new hunt.


Instruction:
Write a Python 3 function that can
schedule foreman rules for a generic hunt.

Response:
def _ScheduleGenericHunt(hunt_obj):
  """Adds foreman rules for a generic hunt."""
  # TODO: Migrate foreman conditions to use relation expiration
  # durations instead of absolute timestamps.
  foreman_condition = foreman_rules.ForemanCondition(
      creation_time=rdfvalue.RDFDatetime.Now(),
      expiration_time=hunt_obj.init_start_time + hunt_obj.duration,
      description="Hunt %s %s" % (hunt_obj.hunt_id, hunt_obj.args.hunt_type),
      client_rule_set=hunt_obj.client_rule_set,
      hunt_id=hunt_obj.hunt_id)

  # Make sure the rule makes sense.
  foreman_condition.Validate()

  data_store.REL_DB.WriteForemanRule(foreman_condition)


Instruction:
Can you generate the documentation for the following Python 3 function
def _ScheduleVariableHunt(hunt_obj):
  if hunt_obj.client_rate != 0:
    raise VariableHuntCanNotHaveClientRateError(hunt_obj.hunt_id,
                                                hunt_obj.client_rate)

  seen_clients = set()
  for flow_group in hunt_obj.args.variable.flow_groups:
    for client_id in flow_group.client_ids:
      if client_id in seen_clients:
        raise CanStartAtMostOneFlowPerClientError(hunt_obj.hunt_id, client_id)
      seen_clients.add(client_id)

  now = rdfvalue.RDFDatetime.Now()
  for flow_group in hunt_obj.args.variable.flow_groups:
    flow_cls = registry.FlowRegistry.FlowClassByName(flow_group.flow_name)
    flow_args = flow_group.flow_args if flow_group.HasField(
        "flow_args") else None

    for client_id in flow_group.client_ids:
      flow.StartFlow(
          client_id=client_id,
          creator=hunt_obj.creator,
          cpu_limit=hunt_obj.per_client_cpu_limit,
          network_bytes_limit=hunt_obj.per_client_network_bytes_limit,
          flow_cls=flow_cls,
          flow_args=flow_args,
          # Setting start_at explicitly ensures that flow.StartFlow won't
          # process flow's Start state right away. Only the flow request
          # will be scheduled.
          start_at=now,
          parent_hunt_id=hunt_obj.hunt_id)

Response: Schedules flows for a variable hunt.


Instruction:
Can you tell what is the following Python 3 function doing
def StartHunt(hunt_id):

  hunt_obj = data_store.REL_DB.ReadHuntObject(hunt_id)
  num_hunt_clients = data_store.REL_DB.CountHuntFlows(hunt_id)

  if hunt_obj.hunt_state != hunt_obj.HuntState.PAUSED:
    raise OnlyPausedHuntCanBeStartedError(hunt_obj)

  data_store.REL_DB.UpdateHuntObject(
      hunt_id,
      hunt_state=hunt_obj.HuntState.STARTED,
      start_time=rdfvalue.RDFDatetime.Now(),
      num_clients_at_start_time=num_hunt_clients,
  )
  hunt_obj = data_store.REL_DB.ReadHuntObject(hunt_id)

  if hunt_obj.args.hunt_type == hunt_obj.args.HuntType.STANDARD:
    _ScheduleGenericHunt(hunt_obj)
  elif hunt_obj.args.hunt_type == hunt_obj.args.HuntType.VARIABLE:
    _ScheduleVariableHunt(hunt_obj)
  else:
    raise UnknownHuntTypeError("Invalid hunt type for hunt %s: %r" %
                               (hunt_id, hunt_obj.args.hunt_type))

  return hunt_obj

Response: Starts a hunt with a given id.


Instruction:
How would you implement a function in Python 3 that
pauses a hunt with a given id.

Response:
def PauseHunt(hunt_id, reason=None):
  """Pauses a hunt with a given id."""

  hunt_obj = data_store.REL_DB.ReadHuntObject(hunt_id)
  if hunt_obj.hunt_state != hunt_obj.HuntState.STARTED:
    raise OnlyStartedHuntCanBePausedError(hunt_obj)

  data_store.REL_DB.UpdateHuntObject(
      hunt_id, hunt_state=hunt_obj.HuntState.PAUSED, hunt_state_comment=reason)
  data_store.REL_DB.RemoveForemanRule(hunt_id=hunt_obj.hunt_id)

  return data_store.REL_DB.ReadHuntObject(hunt_id)


Instruction:
Here you have a function in Python 3, explain what it does
def StopHunt(hunt_id, reason=None):

  hunt_obj = data_store.REL_DB.ReadHuntObject(hunt_id)
  if hunt_obj.hunt_state not in [
      hunt_obj.HuntState.STARTED, hunt_obj.HuntState.PAUSED
  ]:
    raise OnlyStartedOrPausedHuntCanBeStoppedError(hunt_obj)

  data_store.REL_DB.UpdateHuntObject(
      hunt_id, hunt_state=hunt_obj.HuntState.STOPPED, hunt_state_comment=reason)
  data_store.REL_DB.RemoveForemanRule(hunt_id=hunt_obj.hunt_id)

  if (reason is not None and
      hunt_obj.creator not in aff4_users.GRRUser.SYSTEM_USERS):
    notification.Notify(
        hunt_obj.creator, rdf_objects.UserNotification.Type.TYPE_HUNT_STOPPED,
        reason,
        rdf_objects.ObjectReference(
            reference_type=rdf_objects.ObjectReference.Type.HUNT,
            hunt=rdf_objects.HuntReference(hunt_id=hunt_obj.hunt_id)))

  return data_store.REL_DB.ReadHuntObject(hunt_id)

Response: Stops a hunt with a given id.


Instruction:
Make a summary of the following Python 3 code
def UpdateHunt(hunt_id, client_limit=None, client_rate=None, duration=None):

  hunt_obj = data_store.REL_DB.ReadHuntObject(hunt_id)
  if hunt_obj.hunt_state != hunt_obj.HuntState.PAUSED:
    raise OnlyPausedHuntCanBeModifiedError(hunt_obj)

  data_store.REL_DB.UpdateHuntObject(
      hunt_id,
      client_limit=client_limit,
      client_rate=client_rate,
      duration=duration)
  return data_store.REL_DB.ReadHuntObject(hunt_id)

Response: Updates a hunt (it must be paused to be updated).


Instruction:
Can you tell what is the following Python 3 function doing
def StartHuntFlowOnClient(client_id, hunt_id):

  hunt_obj = data_store.REL_DB.ReadHuntObject(hunt_id)
  hunt_obj = CompleteHuntIfExpirationTimeReached(hunt_obj)
  # There may be a little race between foreman rules being removed and
  # foreman scheduling a client on an (already) paused hunt. Making sure
  # we don't lose clients in such a race by accepting clients for paused
  # hunts.
  if not rdf_hunt_objects.IsHuntSuitableForFlowProcessing(hunt_obj.hunt_state):
    return

  if hunt_obj.args.hunt_type == hunt_obj.args.HuntType.STANDARD:
    hunt_args = hunt_obj.args.standard

    if hunt_obj.client_rate > 0:
      # Given that we use caching in _GetNumClients and hunt_obj may be updated
      # in another process, we have to account for cases where num_clients_diff
      # may go below 0.
      num_clients_diff = max(
          0,
          _GetNumClients(hunt_obj.hunt_id) - hunt_obj.num_clients_at_start_time)
      next_client_due_msecs = int(
          num_clients_diff / hunt_obj.client_rate * 60e6)

      start_at = rdfvalue.RDFDatetime.FromMicrosecondsSinceEpoch(
          hunt_obj.last_start_time.AsMicrosecondsSinceEpoch() +
          next_client_due_msecs)
    else:
      start_at = None

    # TODO(user): remove client_rate support when AFF4 is gone.
    # In REL_DB always work as if client rate is 0.

    flow_cls = registry.FlowRegistry.FlowClassByName(hunt_args.flow_name)
    flow_args = hunt_args.flow_args if hunt_args.HasField("flow_args") else None
    flow.StartFlow(
        client_id=client_id,
        creator=hunt_obj.creator,
        cpu_limit=hunt_obj.per_client_cpu_limit,
        network_bytes_limit=hunt_obj.per_client_network_bytes_limit,
        flow_cls=flow_cls,
        flow_args=flow_args,
        start_at=start_at,
        parent_hunt_id=hunt_id)

    if hunt_obj.client_limit:
      if _GetNumClients(hunt_obj.hunt_id) >= hunt_obj.client_limit:
        PauseHunt(hunt_id)

  elif hunt_obj.args.hunt_type == hunt_obj.args.HuntType.VARIABLE:
    raise NotImplementedError()
  else:
    raise UnknownHuntTypeError("Can't determine hunt type when starting "
                               "hunt %s on client %s." % (client_id, hunt_id))

Response: Starts a flow corresponding to a given hunt on a given client.


Instruction:
Write a Python 3 script to
check if the hunt has expired.

Response:
def expired(self):
    """Checks if the hunt has expired."""
    expiry_time = self.expiry_time
    if expiry_time is not None:
      return expiry_time < rdfvalue.RDFDatetime.Now()
    else:
      return False


Instruction:
Make a summary of the following Python 3 code
def GetApprovalForObject(object_urn, token=None, username=""):
    if token is None:
      raise access_control.UnauthorizedAccess(
          "No token given, cannot authenticate.")

    if not username:
      username = token.username

    approvals_root_urn = aff4.ROOT_URN.Add("ACL").Add(
        object_urn.Path()).Add(username)

    children_urns = list(aff4.FACTORY.ListChildren(approvals_root_urn))
    if not children_urns:
      raise access_control.UnauthorizedAccess(
          "No approval found for user %s" % utils.SmartStr(username),
          subject=object_urn)

    last_error = None
    approvals = aff4.FACTORY.MultiOpen(
        children_urns,
        mode="r",
        aff4_type=Approval,
        age=aff4.ALL_TIMES,
        token=token)
    for approval in approvals:
      try:
        test_token = access_control.ACLToken(
            username=username, reason=approval.Get(approval.Schema.REASON))
        approval.CheckAccess(test_token)

        return test_token
      except access_control.UnauthorizedAccess as e:
        last_error = e

    if last_error:
      # We tried all possible approvals, but got no usable results.
      raise access_control.UnauthorizedAccess(last_error, subject=object_urn)
    else:
      # If last error is None, means that none of the URNs in children_urns
      # could be opened. This shouldn't really happen ever, but we have
      # to make sure to provide a meaningful error message.
      raise access_control.UnauthorizedAccess(
          "Couldn't open any of %d approvals "
          "for user %s" % (len(children_urns), utils.SmartStr(username)),
          subject=object_urn)

Response: Returns a list of tokens that can be used to access an object.


Instruction:
Can you create a Python 3 function that
enforces a dual approver policy for access.

Response:
def CheckAccess(self, token):
    """Enforce a dual approver policy for access."""
    namespace, _ = self.urn.Split(2)

    if namespace != "ACL":
      raise access_control.UnauthorizedAccess(
          "Approval object has invalid urn %s." % self.urn,
          subject=self.urn,
          requested_access=token.requested_access)

    user, subject_urn = self.InferUserAndSubjectFromUrn()
    if user != token.username:
      raise access_control.UnauthorizedAccess(
          "Approval object is not for user %s." % token.username,
          subject=self.urn,
          requested_access=token.requested_access)

    now = rdfvalue.RDFDatetime.Now()

    # Is this an emergency access?
    break_glass = self.Get(self.Schema.BREAK_GLASS)
    if break_glass and now < break_glass:
      # This tags the token as an emergency token.
      token.is_emergency = True
      return True

    # Check that there are enough approvers.
    approvers = self.GetNonExpiredApprovers()
    approvers_required = config.CONFIG["ACL.approvers_required"]
    if len(approvers) < approvers_required:
      missing = approvers_required - len(approvers)
      msg = ("Need at least %d additional approver%s for access." %
             (missing, "s" if missing > 1 else ""))

      raise access_control.UnauthorizedAccess(
          msg, subject=subject_urn, requested_access=token.requested_access)

    # Check User labels
    if self.checked_approvers_label:
      approvers_with_label = []

      # We need to check labels with high privilege since normal users can
      # inspect other user's labels.
      for approver in approvers:
        try:
          user = aff4.FACTORY.Open(
              "aff4:/users/%s" % approver,
              aff4_type=aff4_users.GRRUser,
              token=token.SetUID())
          if self.checked_approvers_label in user.GetLabelsNames():
            approvers_with_label.append(approver)
        except IOError:
          pass

      if len(approvers_with_label) < self.min_approvers_with_label:
        missing = self.min_approvers_with_label - len(approvers_with_label)
        raise access_control.UnauthorizedAccess(
            "Need at least 1 admin approver for access.",
            subject=subject_urn,
            requested_access=token.requested_access)

    return True


Instruction:
Write a Python 3 script for
returning a list of usernames of approvers who approved this approval.

Response:
def GetNonExpiredApprovers(self):
    """Returns a list of usernames of approvers who approved this approval."""

    lifetime = rdfvalue.Duration(
        self.Get(self.Schema.LIFETIME) or config.CONFIG["ACL.token_expiry"])

    # Check that there are enough approvers.
    #
    # TODO(user): approvals have to be opened with
    # age=aff4.ALL_TIMES because versioning is used to store lists
    # of approvers. This doesn't seem right and has to be fixed.
    approvers = set()
    now = rdfvalue.RDFDatetime.Now()
    for approver in self.GetValuesForAttribute(self.Schema.APPROVER):
      if approver.age + lifetime > now:
        approvers.add(utils.SmartStr(approver))

    return list(approvers)


Instruction:
Create a Python 3 function to
infer user name and subject urn from self. urn.

Response:
def InferUserAndSubjectFromUrn(self):
    """Infers user name and subject urn from self.urn."""
    _, client_id, user, _ = self.urn.Split(4)
    return (user, rdf_client.ClientURN(client_id))


Instruction:
How would you implement a function in Python 3 that
infers user name and subject urn from self. urn.

Response:
def InferUserAndSubjectFromUrn(self):
    """Infers user name and subject urn from self.urn."""
    _, hunts_str, hunt_id, user, _ = self.urn.Split(5)

    if hunts_str != "hunts":
      raise access_control.UnauthorizedAccess(
          "Approval object has invalid urn %s." % self.urn,
          requested_access=self.token.requested_access)

    return (user, aff4.ROOT_URN.Add("hunts").Add(hunt_id))


Instruction:
How would you code a function in Python 3 to
infer user name and subject urn from self. urn.

Response:
def InferUserAndSubjectFromUrn(self):
    """Infers user name and subject urn from self.urn."""
    _, cron_str, cron_job_name, user, _ = self.urn.Split(5)

    if cron_str != "cron":
      raise access_control.UnauthorizedAccess(
          "Approval object has invalid urn %s." % self.urn,
          requested_access=self.token.requested_access)

    return (user, aff4.ROOT_URN.Add("cron").Add(cron_job_name))


Instruction:
Can you generate a brief explanation for the following Python 3 code
def ApprovalUrnBuilder(subject, user, approval_id):
    return aff4.ROOT_URN.Add("ACL").Add(subject).Add(user).Add(approval_id)

Response: Encode an approval URN.


Instruction:
Explain what the following Python 3 code does
def ApprovalSymlinkUrnBuilder(approval_type, subject_id, user, approval_id):
    return aff4.ROOT_URN.Add("users").Add(user).Add("approvals").Add(
        approval_type).Add(subject_id).Add(approval_id)

Response: Builds an approval symlink URN.


Instruction:
Explain what the following Python 3 code does
def Request(self):

    approval_id = "approval:%X" % random.UInt32()
    approval_urn = self.BuildApprovalUrn(approval_id)

    email_msg_id = email.utils.make_msgid()

    with aff4.FACTORY.Create(
        approval_urn, self.approval_type, mode="w",
        token=self.token) as approval_request:
      approval_request.Set(approval_request.Schema.SUBJECT(self.subject_urn))
      approval_request.Set(
          approval_request.Schema.REQUESTOR(self.token.username))
      approval_request.Set(approval_request.Schema.REASON(self.reason))
      approval_request.Set(approval_request.Schema.EMAIL_MSG_ID(email_msg_id))

      cc_addresses = (self.email_cc_address,
                      config.CONFIG.Get("Email.approval_cc_address"))
      email_cc = ",".join(filter(None, cc_addresses))

      # When we reply with the approval we want to cc all the people to whom the
      # original approval was sent, to avoid people approving stuff that was
      # already approved.
      if email_cc:
        reply_cc = ",".join((self.approver, email_cc))
      else:
        reply_cc = self.approver

      approval_request.Set(approval_request.Schema.EMAIL_CC(reply_cc))

      approval_request.Set(
          approval_request.Schema.NOTIFIED_USERS(self.approver))

      # We add ourselves as an approver as well (The requirement is that we have
      # 2 approvers, so the requester is automatically an approver).
      approval_request.AddAttribute(
          approval_request.Schema.APPROVER(self.token.username))

    approval_link_urns = self.BuildApprovalSymlinksUrns(approval_id)
    for link_urn in approval_link_urns:
      with aff4.FACTORY.Create(
          link_urn, aff4.AFF4Symlink, mode="w", token=self.token) as link:
        link.Set(link.Schema.SYMLINK_TARGET(approval_urn))

    return approval_urn

Response: Create the Approval object and notify the Approval Granter.


Instruction:
Write a Python 3 script to
create the Approval object and notify the Approval Granter.

Response:
def Grant(self):
    """Create the Approval object and notify the Approval Granter."""

    approvals_root_urn = aff4.ROOT_URN.Add("ACL").Add(
        self.subject_urn.Path()).Add(self.delegate)
    children_urns = list(aff4.FACTORY.ListChildren(approvals_root_urn))
    if not children_urns:
      raise access_control.UnauthorizedAccess(
          "No approval found for user %s" % utils.SmartStr(self.token.username),
          subject=self.subject_urn)

    approvals = aff4.FACTORY.MultiOpen(
        children_urns, mode="r", aff4_type=Approval, token=self.token)
    found_approval_urn = None
    for approval in approvals:
      approval_reason = approval.Get(approval.Schema.REASON)
      if (utils.SmartUnicode(approval_reason) == utils.SmartUnicode(self.reason)
          and (not found_approval_urn or
               approval_reason.age > found_approval_urn.age)):
        found_approval_urn = approval.urn
        found_approval_urn.age = approval_reason.age

    if not found_approval_urn:
      raise access_control.UnauthorizedAccess(
          "No approval with reason '%s' found for user %s" % (utils.SmartStr(
              self.reason), utils.SmartStr(self.token.username)),
          subject=self.subject_urn)

    # This object must already exist.
    try:
      approval_request = aff4.FACTORY.Open(
          found_approval_urn,
          mode="rw",
          aff4_type=self.approval_type,
          token=self.token)
    except IOError:
      raise access_control.UnauthorizedAccess(
          "Approval object does not exist.", requested_access="rw")

    with approval_request:
      # We are now an approver for this request.
      approval_request.AddAttribute(
          approval_request.Schema.APPROVER(self.token.username))

    return found_approval_urn


Instruction:
Can you implement a function in Python 3 that
builds approval object urn.

Response:
def BuildApprovalUrn(self, approval_id):
    """Builds approval object urn."""
    event = rdf_events.AuditEvent(
        user=self.token.username,
        action="CLIENT_APPROVAL_REQUEST",
        client=self.subject_urn,
        description=self.reason)
    events.Events.PublishEvent("Audit", event, token=self.token)

    return self.ApprovalUrnBuilder(self.subject_urn.Path(), self.token.username,
                                   approval_id)


Instruction:
Explain what the following Python 3 code does
def BuildApprovalSymlinksUrns(self, approval_id):
    return [
        self.ApprovalSymlinkUrnBuilder("cron", self.subject_urn.Basename(),
                                       self.token.username, approval_id)
    ]

Response: Builds the list of symlinks URNs for the approval object.


Instruction:
Make a summary of the following Python 3 code
def CheckBlobsExist(self, blob_ids):

    result = {}
    for blob_id in blob_ids:
      result[blob_id] = blob_id in self.blobs

    return result

Response: Checks if given blobs exist.


Instruction:
Given the following Python 3 function, write the documentation
def FileFinderOSFromClient(args):
  stat_cache = filesystem.StatCache()

  opts = args.action.stat

  for path in GetExpandedPaths(args):
    try:
      content_conditions = conditions.ContentCondition.Parse(args.conditions)
      for content_condition in content_conditions:
        with io.open(path, "rb") as fd:
          result = list(content_condition.Search(fd))
        if not result:
          raise _SkipFileException()
      # TODO: `opts.resolve_links` has type `RDFBool`, not `bool`.
      stat = stat_cache.Get(path, follow_symlink=bool(opts.resolve_links))
      stat_entry = client_utils.StatEntryFromStatPathSpec(
          stat, ext_attrs=opts.collect_ext_attrs)
      yield stat_entry
    except _SkipFileException:
      pass

Response: This function expands paths from the args and returns related stat entries.


Instruction:
Write a Python 3 script for
returning an iterator that yields absolute paths derived from input patterns.

Response:
def GetExpandedPaths(
    args):
  """Expands given path patterns.

  Args:
    args: A `FileFinderArgs` instance that dictates the behaviour of the path
      expansion.

  Yields:
    Absolute paths (as string objects) derived from input patterns.

  Raises:
    ValueError: For unsupported path types.
  """
  if args.pathtype == rdf_paths.PathSpec.PathType.OS:
    pathtype = rdf_paths.PathSpec.PathType.OS
  else:
    raise ValueError("Unsupported path type: ", args.pathtype)

  opts = globbing.PathOpts(
      follow_links=args.follow_links,
      recursion_blacklist=_GetMountpointBlacklist(args.xdev),
      pathtype=pathtype)

  for path in args.paths:
    for expanded_path in globbing.ExpandPath(str(path), opts):
      yield expanded_path


Instruction:
How would you explain what the following Python 3 function does
def _GetMountpoints(only_physical=True):
  partitions = psutil.disk_partitions(all=not only_physical)
  return set(partition.mountpoint for partition in partitions)

Response: Fetches a list of mountpoints.


Instruction:
Can you tell what is the following Python 3 function doing
def _GetMountpointBlacklist(xdev):
  if xdev == rdf_file_finder.FileFinderArgs.XDev.NEVER:
    # Never cross device boundaries, stop at all mount points.
    return _GetMountpoints(only_physical=False)

  if xdev == rdf_file_finder.FileFinderArgs.XDev.LOCAL:
    # Descend into file systems on physical devices only.
    physical = _GetMountpoints(only_physical=True)
    return _GetMountpoints(only_physical=False) - physical

  if xdev == rdf_file_finder.FileFinderArgs.XDev.ALWAYS:
    # Never stop at any device boundary.
    return set()

  raise ValueError("Incorrect `xdev` value: %s" % xdev)

Response: Returns a list of mountpoints to ignore during recursive searches.


Instruction:
Explain what the following Python 3 code does
def WriteHuntObject(self, hunt_obj):
    if hunt_obj.hunt_id in self.hunts:
      raise db.DuplicatedHuntError(hunt_id=hunt_obj.hunt_id)

    clone = self._DeepCopy(hunt_obj)
    clone.create_time = rdfvalue.RDFDatetime.Now()
    clone.last_update_time = rdfvalue.RDFDatetime.Now()
    self.hunts[(hunt_obj.hunt_id)] = clone

Response: Writes a hunt object to the database.


Instruction:
Can you generate the documentation for the following Python 3 function
def UpdateHuntObject(self, hunt_id, start_time=None, **kwargs):
    hunt_obj = self.ReadHuntObject(hunt_id)

    delta_suffix = "_delta"

    for k, v in kwargs.items():
      if v is None:
        continue

      if k.endswith(delta_suffix):
        key = k[:-len(delta_suffix)]
        current_value = getattr(hunt_obj, key)
        setattr(hunt_obj, key, current_value + v)
      else:
        setattr(hunt_obj, k, v)

    if start_time is not None:
      hunt_obj.init_start_time = hunt_obj.init_start_time or start_time
      hunt_obj.last_start_time = start_time

    hunt_obj.last_update_time = rdfvalue.RDFDatetime.Now()
    self.hunts[hunt_obj.hunt_id] = hunt_obj

Response: Updates the hunt object by applying the update function.


Instruction:
Can you write a function in Python 3 where it
updates hunt output plugin state for a given hunt.

Response:
def UpdateHuntOutputPluginState(self, hunt_id, state_index, update_fn):
    """Updates hunt output plugin state for a given output plugin."""

    if hunt_id not in self.hunts:
      raise db.UnknownHuntError(hunt_id)

    try:
      state = rdf_flow_runner.OutputPluginState.FromSerializedString(
          self.hunt_output_plugins_states[hunt_id][state_index])
    except KeyError:
      raise db.UnknownHuntOutputPluginError(hunt_id, state_index)

    state.plugin_state = update_fn(state.plugin_state)

    self.hunt_output_plugins_states[hunt_id][
        state_index] = state.SerializeToString()

    return state.plugin_state


Instruction:
Implement a function in Python 3 to
read a hunt object from the database.

Response:
def ReadHuntObject(self, hunt_id):
    """Reads a hunt object from the database."""
    try:
      return self._DeepCopy(self.hunts[hunt_id])
    except KeyError:
      raise db.UnknownHuntError(hunt_id)


Instruction:
How would you code a function in Python 3 to
read all hunt objects from the database.

Response:
def ReadHuntObjects(self,
                      offset,
                      count,
                      with_creator=None,
                      created_after=None,
                      with_description_match=None):
    """Reads all hunt objects from the database."""
    filter_fns = []
    if with_creator is not None:
      filter_fns.append(lambda h: h.creator == with_creator)
    if created_after is not None:
      filter_fns.append(lambda h: h.create_time > created_after)
    if with_description_match is not None:
      filter_fns.append(lambda h: with_description_match in h.description)
    filter_fn = lambda h: all(f(h) for f in filter_fns)

    result = [self._DeepCopy(h) for h in self.hunts.values() if filter_fn(h)]
    return sorted(
        result, key=lambda h: h.create_time,
        reverse=True)[offset:offset + (count or db.MAX_COUNT)]


Instruction:
Create a Python 3 function for
reading hunt log entries of a given hunt.

Response:
def ReadHuntLogEntries(self, hunt_id, offset, count, with_substring=None):
    """Reads hunt log entries of a given hunt using given query options."""
    all_entries = []
    for flow_obj in self._GetHuntFlows(hunt_id):
      for entry in self.ReadFlowLogEntries(
          flow_obj.client_id,
          flow_obj.flow_id,
          0,
          sys.maxsize,
          with_substring=with_substring):

        all_entries.append(
            rdf_flow_objects.FlowLogEntry(
                hunt_id=hunt_id,
                client_id=flow_obj.client_id,
                flow_id=flow_obj.flow_id,
                timestamp=entry.timestamp,
                message=entry.message))

    return sorted(all_entries, key=lambda x: x.timestamp)[offset:offset + count]


Instruction:
Write a Python 3 function that can
read hunt results of a given hunt.

Response:
def ReadHuntResults(self,
                      hunt_id,
                      offset,
                      count,
                      with_tag=None,
                      with_type=None,
                      with_substring=None,
                      with_timestamp=None):
    """Reads hunt results of a given hunt using given query options."""
    all_results = []
    for flow_obj in self._GetHuntFlows(hunt_id):
      for entry in self.ReadFlowResults(
          flow_obj.client_id,
          flow_obj.flow_id,
          0,
          sys.maxsize,
          with_tag=with_tag,
          with_type=with_type,
          with_substring=with_substring):
        all_results.append(
            rdf_flow_objects.FlowResult(
                hunt_id=hunt_id,
                client_id=flow_obj.client_id,
                flow_id=flow_obj.flow_id,
                timestamp=entry.timestamp,
                tag=entry.tag,
                payload=entry.payload))

    if with_timestamp:
      all_results = [r for r in all_results if r.timestamp == with_timestamp]

    return sorted(all_results, key=lambda x: x.timestamp)[offset:offset + count]


Instruction:
Write a Python 3 function that can
count hunt results of a given hunt using given query options.

Response:
def CountHuntResults(self, hunt_id, with_tag=None, with_type=None):
    """Counts hunt results of a given hunt using given query options."""
    return len(
        self.ReadHuntResults(
            hunt_id, 0, sys.maxsize, with_tag=with_tag, with_type=with_type))


Instruction:
Write a Python 3 function that can
read hunt flows matching given conditins.

Response:
def ReadHuntFlows(self,
                    hunt_id,
                    offset,
                    count,
                    filter_condition=db.HuntFlowsCondition.UNSET):
    """Reads hunt flows matching given conditins."""
    if filter_condition == db.HuntFlowsCondition.UNSET:
      filter_fn = lambda _: True
    elif filter_condition == db.HuntFlowsCondition.FAILED_FLOWS_ONLY:
      filter_fn = lambda f: f.flow_state == f.FlowState.ERROR
    elif filter_condition == db.HuntFlowsCondition.SUCCEEDED_FLOWS_ONLY:
      filter_fn = lambda f: f.flow_state == f.FlowState.FINISHED
    elif filter_condition == db.HuntFlowsCondition.COMPLETED_FLOWS_ONLY:
      filter_fn = (
          lambda f: f.flow_state in [f.FlowState.ERROR, f.FlowState.FINISHED])
    elif filter_condition == db.HuntFlowsCondition.FLOWS_IN_PROGRESS_ONLY:
      filter_fn = lambda f: f.flow_state == f.FlowState.RUNNING
    elif filter_condition == db.HuntFlowsCondition.CRASHED_FLOWS_ONLY:
      filter_fn = lambda f: f.flow_state == f.FlowState.CRASHED
    else:
      raise ValueError("Invalid filter condition: %d" % filter_condition)

    results = [
        flow_obj for flow_obj in self._GetHuntFlows(hunt_id)
        if filter_fn(flow_obj)
    ]
    results.sort(key=lambda f: f.last_update_time)
    return results[offset:offset + count]


Instruction:
Write a Python 3 script to
count the number of hunt flows matching given conditions.

Response:
def CountHuntFlows(self,
                     hunt_id,
                     filter_condition=db.HuntFlowsCondition.UNSET):
    """Counts hunt flows matching given conditions."""

    return len(
        self.ReadHuntFlows(
            hunt_id, 0, sys.maxsize, filter_condition=filter_condition))


Instruction:
Can you tell what is the following Python 3 function doing
def ReadHuntClientResourcesStats(self, hunt_id):

    result = rdf_stats.ClientResourcesStats()
    for f in self._GetHuntFlows(hunt_id):
      cr = rdf_client_stats.ClientResources(
          session_id="%s/%s" % (f.client_id, f.flow_id),
          client_id=f.client_id,
          cpu_usage=f.cpu_time_used,
          network_bytes_sent=f.network_bytes_sent)
      result.RegisterResources(cr)

    # TODO(user): remove this hack when compatibility with AFF4 is not
    # important.
    return rdf_stats.ClientResourcesStats.FromSerializedString(
        result.SerializeToString())

Response: Reads and calculates hunt client resources stats.


Instruction:
Create a Python 3 function to
read hunt flows states and timestamps.

Response:
def ReadHuntFlowsStatesAndTimestamps(self, hunt_id):
    """Reads hunt flows states and timestamps."""

    result = []
    for f in self._GetHuntFlows(hunt_id):
      result.append(
          db.FlowStateAndTimestamps(
              flow_state=f.flow_state,
              create_time=f.create_time,
              last_update_time=f.last_update_time))

    return result


Instruction:
Can you create a Python 3 function that
reads hunt output plugin log entries.

Response:
def ReadHuntOutputPluginLogEntries(self,
                                     hunt_id,
                                     output_plugin_id,
                                     offset,
                                     count,
                                     with_type=None):
    """Reads hunt output plugin log entries."""

    all_entries = []
    for flow_obj in self._GetHuntFlows(hunt_id):
      for entry in self.ReadFlowOutputPluginLogEntries(
          flow_obj.client_id,
          flow_obj.flow_id,
          output_plugin_id,
          0,
          sys.maxsize,
          with_type=with_type):
        all_entries.append(
            rdf_flow_objects.FlowOutputPluginLogEntry(
                hunt_id=hunt_id,
                client_id=flow_obj.client_id,
                flow_id=flow_obj.flow_id,
                output_plugin_id=output_plugin_id,
                log_entry_type=entry.log_entry_type,
                timestamp=entry.timestamp,
                message=entry.message))

    return sorted(all_entries, key=lambda x: x.timestamp)[offset:offset + count]


Instruction:
How would you implement a function in Python 3 that
counts hunt output plugin log entries.

Response:
def CountHuntOutputPluginLogEntries(self,
                                      hunt_id,
                                      output_plugin_id,
                                      with_type=None):
    """Counts hunt output plugin log entries."""

    return len(
        self.ReadHuntOutputPluginLogEntries(
            hunt_id, output_plugin_id, 0, sys.maxsize, with_type=with_type))


Instruction:
Create a Python 3 function to
write a foreman rule to the database.

Response:
def WriteForemanRule(self, rule, cursor=None):
    """Writes a foreman rule to the database."""
    query = ("INSERT INTO foreman_rules "
             "  (hunt_id, expiration_time, rule) "
             "VALUES (%s, FROM_UNIXTIME(%s), %s) "
             "ON DUPLICATE KEY UPDATE "
             "  expiration_time=FROM_UNIXTIME(%s), rule=%s")

    exp_str = mysql_utils.RDFDatetimeToTimestamp(rule.expiration_time),
    rule_str = rule.SerializeToString()
    cursor.execute(query, [rule.hunt_id, exp_str, rule_str, exp_str, rule_str])


Instruction:
How would you code a function in Python 3 to
run the main test harness.

Response:
def main(_):
  """Run the main test harness."""

  if flags.FLAGS.version:
    print("GRR Admin UI {}".format(config_server.VERSION["packageversion"]))
    return

  config.CONFIG.AddContext(
      contexts.ADMIN_UI_CONTEXT,
      "Context applied when running the admin user interface GUI.")
  server_startup.Init()

  if not config.CONFIG["AdminUI.headless"] and (not os.path.exists(
      os.path.join(config.CONFIG["AdminUI.document_root"],
                   "dist/grr-ui.bundle.js")) or not os.path.exists(
                       os.path.join(config.CONFIG["AdminUI.document_root"],
                                    "dist/grr-ui.bundle.css"))):
    raise RuntimeError("Can't find compiled JS/CSS bundles. "
                       "Please reinstall the PIP package using "
                       "\"pip install -e .\" to rebuild the bundles.")

  # Start up a server in another thread
  bind_address = config.CONFIG["AdminUI.bind"]
  ip = ipaddress.ip_address(bind_address)
  if ip.version == 4:
    # Address looks like an IPv4 address.
    ThreadedServer.address_family = socket.AF_INET

  max_port = config.CONFIG.Get("AdminUI.port_max",
                               config.CONFIG["AdminUI.port"])

  for port in range(config.CONFIG["AdminUI.port"], max_port + 1):
    # Make a simple reference implementation WSGI server
    try:
      server = simple_server.make_server(
          bind_address,
          port,
          wsgiapp.AdminUIApp().WSGIHandler(),
          server_class=ThreadedServer)
      break
    except socket.error as e:
      if e.errno == socket.errno.EADDRINUSE and port < max_port:
        logging.info("Port %s in use, trying %s", port, port + 1)
      else:
        raise

  proto = "HTTP"

  if config.CONFIG["AdminUI.enable_ssl"]:
    cert_file = config.CONFIG["AdminUI.ssl_cert_file"]
    if not cert_file:
      raise ValueError("Need a valid cert file to enable SSL.")

    key_file = config.CONFIG["AdminUI.ssl_key_file"]
    server.socket = ssl.wrap_socket(
        server.socket, certfile=cert_file, keyfile=key_file, server_side=True)
    proto = "HTTPS"

    # SSL errors are swallowed by the WSGIServer so if your configuration does
    # not work, uncomment the line below, point your browser at the gui and look
    # at the log file to see why SSL complains:
    # server.socket.accept()

  sa = server.socket.getsockname()
  logging.info("Serving %s on %s port %d ...", proto, sa[0], sa[1])
  server_startup.DropPrivileges()

  server.serve_forever()


Instruction:
Can you generate a brief explanation for the following Python 3 code
def WriteAllCrashDetails(client_id,
                         crash_details,
                         flow_session_id=None,
                         hunt_session_id=None,
                         token=None):
  """Updates the last crash attribute of the client."""
  # AFF4.
  if data_store.AFF4Enabled():
    with aff4.FACTORY.Create(
        client_id, aff4_grr.VFSGRRClient, token=token) as client_obj:
      client_obj.Set(client_obj.Schema.LAST_CRASH(crash_details))

    # Duplicate the crash information in a number of places so we can find it
    # easily.
    client_urn = rdf_client.ClientURN(client_id)
    client_crashes = aff4_grr.VFSGRRClient.CrashCollectionURNForCID(client_urn)
    with data_store.DB.GetMutationPool() as pool:
      grr_collections.CrashCollection.StaticAdd(
          client_crashes, crash_details, mutation_pool=pool)

  # Relational db.
  if data_store.RelationalDBEnabled():
    try:
      data_store.REL_DB.WriteClientCrashInfo(client_id, crash_details)
    except db.UnknownClientError:
      pass

  if not flow_session_id:
    return

  if data_store.RelationalDBEnabled():
    flow_id = flow_session_id.Basename()
    data_store.REL_DB.UpdateFlow(
        client_id, flow_id, client_crash_info=crash_details)

    flow_obj = data_store.REL_DB.ReadFlowObject(client_id, flow_id)
    if flow_obj.parent_hunt_id:
      db_compat.ProcessHuntClientCrash(
          flow_obj, client_crash_info=crash_details)

  # TODO(amoser): Registering crashes in hunts is currently not implemented for
  # the relational db.
  if not data_store.RelationalDBEnabled():
    with aff4.FACTORY.Open(
        flow_session_id,
        flow.GRRFlow,
        mode="rw",
        age=aff4.NEWEST_TIME,
        token=token) as aff4_flow:
      aff4_flow.Set(aff4_flow.Schema.CLIENT_CRASH(crash_details))

    hunt_session_id = ExtractHuntId(flow_session_id)
    if hunt_session_id and hunt_session_id != flow_session_id:
      hunt_obj = aff4.FACTORY.Open(
          hunt_session_id,
          aff4_type=implementation.GRRHunt,
          mode="rw",
          token=token)
      hunt_obj.RegisterCrash(crash_details)

Response: Writes the last crash information for a given client.


Instruction:
How would you explain what the following Python 3 function does
def ProcessMessages(self, msgs=None, token=None):
    nanny_msg = ""

    for crash_details in msgs:
      client_urn = crash_details.client_id
      client_id = client_urn.Basename()

      # The session id of the flow that crashed.
      session_id = crash_details.session_id

      # Log.
      logging.info("Client crash reported, client %s.", client_urn)

      # Export.
      stats_collector_instance.Get().IncrementCounter("grr_client_crashes")

      # Write crash data.
      if data_store.RelationalDBEnabled():
        client = data_store.REL_DB.ReadClientSnapshot(client_id)
        if client:
          crash_details.client_info = client.startup_info.client_info
          hostname = client.knowledge_base.fqdn
        else:
          hostname = ""

      if data_store.AFF4Enabled():
        client = aff4.FACTORY.Open(client_urn, token=token)
        client_info = client.Get(client.Schema.CLIENT_INFO)
        hostname = client.Get(client.Schema.FQDN)
        if client_info:
          crash_details.client_info = client_info

      crash_details.crash_type = "Client Crash"

      if nanny_msg:
        termination_msg = "Client crashed, " + nanny_msg
      else:
        termination_msg = "Client crashed."

      # Terminate the flow.
      if data_store.RelationalDBEnabled():
        flow_id = session_id.Basename()
        flow_base.TerminateFlow(
            client_id,
            flow_id,
            reason=termination_msg,
            flow_state=rdf_flow_objects.Flow.FlowState.CRASHED)
      else:
        flow.GRRFlow.TerminateAFF4Flow(
            session_id, reason=termination_msg, token=token)

      WriteAllCrashDetails(
          client_id, crash_details, flow_session_id=session_id, token=token)

      # Also send email.
      to_send = []

      try:
        hunt_session_id = ExtractHuntId(session_id)
        if hunt_session_id and hunt_session_id != session_id:

          # TODO(amoser): Enable this for the relational db once we have hunt
          # metadata.
          if data_store.AFF4Enabled():
            hunt_obj = aff4.FACTORY.Open(
                hunt_session_id, aff4_type=implementation.GRRHunt, token=token)
            email = hunt_obj.runner_args.crash_alert_email

          if email:
            to_send.append(email)
      except aff4.InstantiationError:
        logging.error("Failed to open hunt %s.", hunt_session_id)

      email = config.CONFIG["Monitoring.alert_email"]
      if email:
        to_send.append(email)

      for email_address in to_send:
        if crash_details.nanny_status:
          nanny_msg = "Nanny status: %s" % crash_details.nanny_status

        body = self.__class__.mail_template.render(
            client_id=client_id,
            admin_ui=config.CONFIG["AdminUI.url"],
            hostname=utils.SmartUnicode(hostname),
            url="/clients/%s" % client_id,
            nanny_msg=utils.SmartUnicode(nanny_msg),
            signature=config.CONFIG["Email.signature"])

        email_alerts.EMAIL_ALERTER.SendEmail(
            email_address,
            "GRR server",
            "Client %s reported a crash." % client_id,
            utils.SmartStr(body),
            is_html=True)

Response: Processes the given messages.


Instruction:
Implement a function in Python 3 to
process the response of a client.

Response:
def ProcessResponse(self, client_id, response):
    """Actually processes the contents of the response."""
    precondition.AssertType(client_id, Text)
    downsampled = rdf_client_stats.ClientStats.Downsampled(response)

    if data_store.AFF4Enabled():
      urn = rdf_client.ClientURN(client_id).Add("stats")

      with aff4.FACTORY.Create(
          urn, aff4_stats.ClientStats, token=self.token, mode="w") as stats_fd:
        # Only keep the average of all values that fall within one minute.
        stats_fd.AddAttribute(stats_fd.Schema.STATS, downsampled)

    if data_store.RelationalDBEnabled():
      data_store.REL_DB.WriteClientStats(client_id, downsampled)

    return downsampled


Instruction:
Can you write a function in Python 3 where it
processes a stats response from the client.

Response:
def ProcessMessage(self, message):
    """Processes a stats response from the client."""
    self.ProcessResponse(message.source.Basename(), message.payload)


Instruction:
How would you explain what the following Python 3 function does
def ProcessMessage(self, message):
    # Only accept authenticated messages
    if (message.auth_state !=
        rdf_flows.GrrMessage.AuthorizationState.AUTHENTICATED):
      return

    now = time.time()

    # Maintain a cache of the foreman
    with self.lock:
      if (self.foreman_cache is None or
          now > self.foreman_cache.age + self.cache_refresh_time):
        self.foreman_cache = aff4.FACTORY.Open(
            "aff4:/foreman", mode="rw", token=self.token)
        self.foreman_cache.age = now

    if message.source:
      self.foreman_cache.AssignTasksToClient(message.source.Basename())

Response: Process a message from the client.


Instruction:
Write a Python 3 script for
sending an email to the GRR server.

Response:
def SendEmail(self, client_id, message):
    """Processes this event."""
    logging.info(self.logline, client_id, message)

    client_info = None
    hostname = None

    # Write crash data.
    if data_store.RelationalDBEnabled():
      client = data_store.REL_DB.ReadClientSnapshot(client_id)
      if client is not None:
        client_info = client.startup_info.client_info
        hostname = client.knowledge_base.fqdn
    else:
      client = aff4.FACTORY.Open(client_id, token=self.token)
      client_info = client.Get(client.Schema.CLIENT_INFO)
      hostname = client.Get(client.Schema.FQDN)

    crash_details = rdf_client.ClientCrash(
        client_id=client_id,
        client_info=client_info,
        crash_message=message,
        timestamp=int(time.time() * 1e6),
        crash_type="Nanny Message")

    WriteAllCrashDetails(client_id, crash_details, token=self.token)

    # Also send email.
    if config.CONFIG["Monitoring.alert_email"]:
      url = "/clients/%s" % client_id
      body = self.__class__.mail_template.render(
          client_id=client_id,
          admin_ui=config.CONFIG["AdminUI.url"],
          hostname=utils.SmartUnicode(hostname),
          signature=config.CONFIG["Email.signature"],
          url=url,
          message=utils.SmartUnicode(message))
      email_alerts.EMAIL_ALERTER.SendEmail(
          config.CONFIG["Monitoring.alert_email"],
          "GRR server",
          self.subject % client_id,
          utils.SmartStr(body),
          is_html=True)


Instruction:
Explain what the following Python 3 code does
def WriteClientStartupInfo(self, client_id, new_si):
    drift = rdfvalue.Duration("5m")

    if data_store.RelationalDBEnabled():
      current_si = data_store.REL_DB.ReadClientStartupInfo(client_id)

      # We write the updated record if the client_info has any changes
      # or the boot time is more than 5 minutes different.
      if (not current_si or current_si.client_info != new_si.client_info or
          not current_si.boot_time or
          abs(current_si.boot_time - new_si.boot_time) > drift):
        try:
          data_store.REL_DB.WriteClientStartupInfo(client_id, new_si)
        except db.UnknownClientError:
          # On first contact with a new client, this write will fail.
          logging.info("Can't write StartupInfo for unknown client %s",
                       client_id)
    else:
      changes = False
      with aff4.FACTORY.Create(
          client_id, aff4_grr.VFSGRRClient, mode="rw",
          token=self.token) as client:
        old_info = client.Get(client.Schema.CLIENT_INFO)
        old_boot = client.Get(client.Schema.LAST_BOOT_TIME, 0)

        info = new_si.client_info

        # Only write to the datastore if we have new information.
        if info != old_info:
          client.Set(client.Schema.CLIENT_INFO(info))
          changes = True

        client.AddLabels(info.labels, owner="GRR")

        # Allow for some drift in the boot times (5 minutes).
        if not old_boot or abs(old_boot - new_si.boot_time) > drift:
          client.Set(client.Schema.LAST_BOOT_TIME(new_si.boot_time))
          changes = True

      if data_store.RelationalDBEnabled() and changes:
        try:
          data_store.REL_DB.WriteClientStartupInfo(client_id, new_si)
        except db.UnknownClientError:
          pass

Response: Write a startup info record for a given client.


Instruction:
Make a summary of the following Python 3 code
def WriteMessageHandlerRequests(self, requests):
    now = rdfvalue.RDFDatetime.Now()
    for r in requests:
      flow_dict = self.message_handler_requests.setdefault(r.handler_name, {})
      cloned_request = r.Copy()
      cloned_request.timestamp = now
      flow_dict[cloned_request.request_id] = cloned_request

Response: Writes a list of message handler requests to the database.


Instruction:
Write a Python 3 function for
reading all message handler requests from the database.

Response:
def ReadMessageHandlerRequests(self):
    """Reads all message handler requests from the database."""
    res = []
    leases = self.message_handler_leases
    for requests in itervalues(self.message_handler_requests):
      for r in itervalues(requests):
        res.append(r.Copy())
        existing_lease = leases.get(r.handler_name, {}).get(r.request_id, None)
        res[-1].leased_until = existing_lease

    return sorted(res, key=lambda r: -1 * r.timestamp)


Instruction:
How would you explain what the following Python 3 function does
def DeleteMessageHandlerRequests(self, requests):

    for r in requests:
      flow_dict = self.message_handler_requests.get(r.handler_name, {})
      if r.request_id in flow_dict:
        del flow_dict[r.request_id]
      flow_dict = self.message_handler_leases.get(r.handler_name, {})
      if r.request_id in flow_dict:
        del flow_dict[r.request_id]

Response: Deletes a list of message handler requests from the database.


Instruction:
Implement a Python 3 function for
registering a message handler.

Response:
def RegisterMessageHandler(self, handler, lease_time, limit=1000):
    """Leases a number of message handler requests up to the indicated limit."""
    self.UnregisterMessageHandler()

    self.handler_stop = False
    self.handler_thread = threading.Thread(
        name="message_handler",
        target=self._MessageHandlerLoop,
        args=(handler, lease_time, limit))
    self.handler_thread.daemon = True
    self.handler_thread.start()


Instruction:
Create a Python 3 function to
read and lease some outstanding message handler requests.

Response:
def _LeaseMessageHandlerRequests(self, lease_time, limit):
    """Read and lease some outstanding message handler requests."""
    leased_requests = []

    now = rdfvalue.RDFDatetime.Now()
    zero = rdfvalue.RDFDatetime.FromSecondsSinceEpoch(0)
    expiration_time = now + lease_time

    leases = self.message_handler_leases
    for requests in itervalues(self.message_handler_requests):
      for r in itervalues(requests):
        existing_lease = leases.get(r.handler_name, {}).get(r.request_id, zero)
        if existing_lease < now:
          leases.setdefault(r.handler_name, {})[r.request_id] = expiration_time
          r.leased_until = expiration_time
          r.leased_by = utils.ProcessIdString()
          leased_requests.append(r)
          if len(leased_requests) >= limit:
            break

    return leased_requests


Instruction:
Can you generate a brief explanation for the following Python 3 code
def ReadAllClientActionRequests(self, client_id):
    res = []
    for key, orig_request in iteritems(self.client_action_requests):
      request_client_id, _, _ = key
      if request_client_id != client_id:
        continue

      request = orig_request.Copy()
      current_lease = self.client_action_request_leases.get(key)
      request.ttl = db.Database.CLIENT_MESSAGES_TTL
      if current_lease is not None:
        request.leased_until, request.leased_by, leased_count = current_lease
        request.ttl -= leased_count
      else:
        request.leased_until = None
        request.leased_by = None
      res.append(request)

    return res

Response: Reads all client action requests available for a given client_id.


Instruction:
Here you have a function in Python 3, explain what it does
def LeaseClientActionRequests(self,
                                client_id,
                                lease_time=None,
                                limit=sys.maxsize):
    """Leases available client action requests for a client."""

    leased_requests = []

    now = rdfvalue.RDFDatetime.Now()
    expiration_time = now + lease_time
    process_id_str = utils.ProcessIdString()

    leases = self.client_action_request_leases
    # Can't use an iterator here since the dict might change when requests get
    # deleted.
    for key, request in sorted(self.client_action_requests.items()):
      if key[0] != client_id:
        continue

      existing_lease = leases.get(key)
      if not existing_lease or existing_lease[0] < now:
        if existing_lease:
          lease_count = existing_lease[-1] + 1
          if lease_count > db.Database.CLIENT_MESSAGES_TTL:
            self._DeleteClientActionRequest(*key)
            continue
        else:
          lease_count = 1

        leases[key] = (expiration_time, process_id_str, lease_count)
        request.leased_until = expiration_time
        request.leased_by = process_id_str
        request.ttl = db.Database.CLIENT_MESSAGES_TTL - lease_count
        leased_requests.append(request)
        if len(leased_requests) >= limit:
          break

    return leased_requests

Response: Leases available client action requests for a client.


Instruction:
How would you implement a function in Python 3 that
writes messages that should go to the client to the db.

Response:
def WriteClientActionRequests(self, requests):
    """Writes messages that should go to the client to the db."""
    for r in requests:
      req_dict = self.flow_requests.get((r.client_id, r.flow_id), {})
      if r.request_id not in req_dict:
        request_keys = [(r.client_id, r.flow_id, r.request_id) for r in requests
                       ]
        raise db.AtLeastOneUnknownRequestError(request_keys)

    for r in requests:
      request_key = (r.client_id, r.flow_id, r.request_id)
      self.client_action_requests[request_key] = r


Instruction:
Given the following Python 3 function, write the documentation
def WriteFlowObject(self, flow_obj):
    if flow_obj.client_id not in self.metadatas:
      raise db.UnknownClientError(flow_obj.client_id)

    clone = flow_obj.Copy()
    clone.last_update_time = rdfvalue.RDFDatetime.Now()
    self.flows[(flow_obj.client_id, flow_obj.flow_id)] = clone

Response: Writes a flow object to the database.


Instruction:
Write a Python 3 script to
read a flow object from the database.

Response:
def ReadFlowObject(self, client_id, flow_id):
    """Reads a flow object from the database."""
    try:
      return self.flows[(client_id, flow_id)].Copy()
    except KeyError:
      raise db.UnknownFlowError(client_id, flow_id)


Instruction:
Can you write a function in Python 3 where it
reads all flow objects.

Response:
def ReadAllFlowObjects(
      self,
      client_id = None,
      min_create_time = None,
      max_create_time = None,
      include_child_flows = True,
  ):
    """Returns all flow objects."""
    res = []
    for flow in itervalues(self.flows):
      if ((client_id is None or flow.client_id == client_id) and
          (min_create_time is None or flow.create_time >= min_create_time) and
          (max_create_time is None or flow.create_time <= max_create_time) and
          (include_child_flows or not flow.parent_flow_id)):
        res.append(flow.Copy())
    return res


Instruction:
Make a summary of the following Python 3 code
def ReadChildFlowObjects(self, client_id, flow_id):
    res = []
    for flow in itervalues(self.flows):
      if flow.client_id == client_id and flow.parent_flow_id == flow_id:
        res.append(flow)
    return res

Response: Reads child flows that were started by a given flow.


Instruction:
Can you tell what is the following Python 3 function doing
def LeaseFlowForProcessing(self, client_id, flow_id, processing_time):
    rdf_flow = self.ReadFlowObject(client_id, flow_id)
    # TODO(user): remove the check for a legacy hunt prefix as soon as
    # AFF4 is gone.
    if rdf_flow.parent_hunt_id and not rdf_flow.parent_hunt_id.startswith("H:"):
      rdf_hunt = self.ReadHuntObject(rdf_flow.parent_hunt_id)
      if not rdf_hunt_objects.IsHuntSuitableForFlowProcessing(
          rdf_hunt.hunt_state):
        raise db.ParentHuntIsNotRunningError(client_id, flow_id,
                                             rdf_hunt.hunt_id,
                                             rdf_hunt.hunt_state)

    now = rdfvalue.RDFDatetime.Now()
    if rdf_flow.processing_on and rdf_flow.processing_deadline > now:
      raise ValueError("Flow %s on client %s is already being processed." %
                       (client_id, flow_id))
    processing_deadline = now + processing_time
    process_id_string = utils.ProcessIdString()
    self.UpdateFlow(
        client_id,
        flow_id,
        processing_on=process_id_string,
        processing_since=now,
        processing_deadline=processing_deadline)
    rdf_flow.processing_on = process_id_string
    rdf_flow.processing_since = now
    rdf_flow.processing_deadline = processing_deadline
    return rdf_flow

Response: Leases a flow for processing.


Instruction:
Can you generate the documentation for the following Python 3 function
def UpdateFlow(self,
                 client_id,
                 flow_id,
                 flow_obj=db.Database.unchanged,
                 flow_state=db.Database.unchanged,
                 client_crash_info=db.Database.unchanged,
                 pending_termination=db.Database.unchanged,
                 processing_on=db.Database.unchanged,
                 processing_since=db.Database.unchanged,
                 processing_deadline=db.Database.unchanged):
    """Updates flow objects in the database."""

    try:
      flow = self.flows[(client_id, flow_id)]
    except KeyError:
      raise db.UnknownFlowError(client_id, flow_id)

    if flow_obj != db.Database.unchanged:
      self.flows[(client_id, flow_id)] = flow_obj
      flow = flow_obj

    if flow_state != db.Database.unchanged:
      flow.flow_state = flow_state
    if client_crash_info != db.Database.unchanged:
      flow.client_crash_info = client_crash_info
    if pending_termination != db.Database.unchanged:
      flow.pending_termination = pending_termination
    if processing_on != db.Database.unchanged:
      flow.processing_on = processing_on
    if processing_since != db.Database.unchanged:
      flow.processing_since = processing_since
    if processing_deadline != db.Database.unchanged:
      flow.processing_deadline = processing_deadline
    flow.last_update_time = rdfvalue.RDFDatetime.Now()

Response: Updates the flow with the given client_id and flow_id.


Instruction:
How would you explain what the following Python 3 function does
def UpdateFlows(self,
                  client_id_flow_id_pairs,
                  pending_termination=db.Database.unchanged):
    """Updates flow objects in the database."""
    for client_id, flow_id in client_id_flow_id_pairs:
      try:
        self.UpdateFlow(
            client_id, flow_id, pending_termination=pending_termination)
      except db.UnknownFlowError:
        pass

Response: Updates the flow objects in the database.


Instruction:
Create a Python 3 function for
writing a list of flow requests to the database.

Response:
def WriteFlowRequests(self, requests):
    """Writes a list of flow requests to the database."""
    flow_processing_requests = []

    for request in requests:
      if (request.client_id, request.flow_id) not in self.flows:
        raise db.AtLeastOneUnknownFlowError([(request.client_id,
                                              request.flow_id)])

    for request in requests:
      key = (request.client_id, request.flow_id)
      request_dict = self.flow_requests.setdefault(key, {})
      request_dict[request.request_id] = request.Copy()
      request_dict[request.request_id].timestamp = rdfvalue.RDFDatetime.Now()

      if request.needs_processing:
        flow = self.flows[(request.client_id, request.flow_id)]
        if flow.next_request_to_process == request.request_id:
          flow_processing_requests.append(
              rdf_flows.FlowProcessingRequest(
                  client_id=request.client_id,
                  flow_id=request.flow_id,
                  delivery_time=request.start_time))

    if flow_processing_requests:
      self.WriteFlowProcessingRequests(flow_processing_requests)


Instruction:
How would you explain what the following Python 3 function does
def DeleteFlowRequests(self, requests):
    for request in requests:
      if (request.client_id, request.flow_id) not in self.flows:
        raise db.UnknownFlowError(request.client_id, request.flow_id)

    for request in requests:
      key = (request.client_id, request.flow_id)
      request_dict = self.flow_requests.get(key, {})
      try:
        del request_dict[request.request_id]
      except KeyError:
        raise db.UnknownFlowRequestError(request.client_id, request.flow_id,
                                         request.request_id)

      response_dict = self.flow_responses.get(key, {})
      try:
        del response_dict[request.request_id]
      except KeyError:
        pass

Response: Deletes a list of flow requests from the database.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def WriteFlowResponses(self, responses):
    status_available = set()
    requests_updated = set()
    task_ids_by_request = {}

    for response in responses:
      flow_key = (response.client_id, response.flow_id)
      if flow_key not in self.flows:
        logging.error("Received response for unknown flow %s, %s.",
                      response.client_id, response.flow_id)
        continue

      request_dict = self.flow_requests.get(flow_key, {})
      if response.request_id not in request_dict:
        logging.error("Received response for unknown request %s, %s, %d.",
                      response.client_id, response.flow_id, response.request_id)
        continue

      response_dict = self.flow_responses.setdefault(flow_key, {})
      clone = response.Copy()
      clone.timestamp = rdfvalue.RDFDatetime.Now()

      response_dict.setdefault(response.request_id,
                               {})[response.response_id] = clone

      if isinstance(response, rdf_flow_objects.FlowStatus):
        status_available.add(response)

      request_key = (response.client_id, response.flow_id, response.request_id)
      requests_updated.add(request_key)
      try:
        task_ids_by_request[request_key] = response.task_id
      except AttributeError:
        pass

    # Every time we get a status we store how many responses are expected.
    for status in status_available:
      request_dict = self.flow_requests[(status.client_id, status.flow_id)]
      request = request_dict[status.request_id]
      request.nr_responses_expected = status.response_id

    # And we check for all updated requests if we need to process them.
    needs_processing = []
    for client_id, flow_id, request_id in requests_updated:
      flow_key = (client_id, flow_id)
      request_dict = self.flow_requests[flow_key]
      request = request_dict[request_id]
      if request.nr_responses_expected and not request.needs_processing:
        response_dict = self.flow_responses.setdefault(flow_key, {})
        responses = response_dict.get(request_id, {})

        if len(responses) == request.nr_responses_expected:
          request.needs_processing = True
          self._DeleteClientActionRequest(client_id, flow_id, request_id)

          flow = self.flows[flow_key]
          if flow.next_request_to_process == request_id:
            needs_processing.append(
                rdf_flows.FlowProcessingRequest(
                    client_id=client_id, flow_id=flow_id))

    if needs_processing:
      self.WriteFlowProcessingRequests(needs_processing)

Response: Writes FlowMessages and updates corresponding requests.


Instruction:
Here you have a function in Python 3, explain what it does
def ReadAllFlowRequestsAndResponses(self, client_id, flow_id):
    flow_key = (client_id, flow_id)
    try:
      self.flows[flow_key]
    except KeyError:
      return []

    request_dict = self.flow_requests.get(flow_key, {})
    response_dict = self.flow_responses.get(flow_key, {})

    res = []
    for request_id in sorted(request_dict):
      res.append((request_dict[request_id], response_dict.get(request_id, {})))

    return res

Response: Reads all requests and responses for a given flow from the database.


Instruction:
Can you tell what is the following Python 3 function doing
def DeleteAllFlowRequestsAndResponses(self, client_id, flow_id):
    flow_key = (client_id, flow_id)
    try:
      self.flows[flow_key]
    except KeyError:
      raise db.UnknownFlowError(client_id, flow_id)

    try:
      del self.flow_requests[flow_key]
    except KeyError:
      pass

    try:
      del self.flow_responses[flow_key]
    except KeyError:
      pass

Response: Deletes all requests and responses for a given flow from the database.


Instruction:
Write a Python 3 script for
reading all requests for a flow that can be processed by the worker.

Response:
def ReadFlowRequestsReadyForProcessing(self,
                                         client_id,
                                         flow_id,
                                         next_needed_request=None):
    """Reads all requests for a flow that can be processed by the worker."""
    request_dict = self.flow_requests.get((client_id, flow_id), {})
    response_dict = self.flow_responses.get((client_id, flow_id), {})

    res = {}
    for request_id in sorted(request_dict):
      # Ignore outdated requests.
      if request_id < next_needed_request:
        continue
      # The request we are currently looking for is not in yet, we are done.
      if request_id != next_needed_request:
        break
      request = request_dict[request_id]
      if not request.needs_processing:
        break

      responses = sorted(
          itervalues(response_dict.get(request_id, {})),
          key=lambda response: response.response_id)
      # Serialize/deserialize responses to better simulate the
      # real DB behavior (where serialization/deserialization is almost
      # guaranteed to be done).
      # TODO(user): change mem-db implementation to do
      # serialization/deserialization everywhere in a generic way.
      responses = [
          r.__class__.FromSerializedString(r.SerializeToString())
          for r in responses
      ]
      res[request_id] = (request, responses)
      next_needed_request += 1

    return res


Instruction:
Write a Python 3 script to
release a flow that the worker was processing.

Response:
def ReleaseProcessedFlow(self, flow_obj):
    """Releases a flow that the worker was processing to the database."""
    key = (flow_obj.client_id, flow_obj.flow_id)
    next_id_to_process = flow_obj.next_request_to_process
    request_dict = self.flow_requests.get(key, {})
    if (next_id_to_process in request_dict and
        request_dict[next_id_to_process].needs_processing):
      return False

    self.UpdateFlow(
        flow_obj.client_id,
        flow_obj.flow_id,
        flow_obj=flow_obj,
        processing_on=None,
        processing_since=None,
        processing_deadline=None)
    return True


Instruction:
Implement a function in Python 3 to
write a list of flow processing requests to the database.

Response:
def WriteFlowProcessingRequests(self, requests):
    """Writes a list of flow processing requests to the database."""
    # If we don't have a handler thread running, we might be able to process the
    # requests inline. If we are not, we start the handler thread for real and
    # queue the requests normally.
    if not self.flow_handler_thread and self.flow_handler_target:
      if self._InlineProcessingOK(requests):
        for r in requests:
          self.flow_handler_target(r)
        return
      else:
        self._RegisterFlowProcessingHandler(self.flow_handler_target)
        self.flow_handler_target = None

    now = rdfvalue.RDFDatetime.Now()
    for r in requests:
      cloned_request = r.Copy()
      cloned_request.timestamp = now
      key = (r.client_id, r.flow_id)
      self.flow_processing_requests[key] = cloned_request


Instruction:
Implement a Python 3 function for
acknowledging flow processing requests from the database.

Response:
def AckFlowProcessingRequests(self, requests):
    """Deletes a list of flow processing requests from the database."""
    for r in requests:
      key = (r.client_id, r.flow_id)
      if key in self.flow_processing_requests:
        del self.flow_processing_requests[key]


Instruction:
Can you create a Python 3 function that
registers a handler to receive flow processing messages.

Response:
def _RegisterFlowProcessingHandler(self, handler):
    """Registers a handler to receive flow processing messages."""
    self.flow_handler_stop = False
    self.flow_handler_thread = threading.Thread(
        name="flow_processing_handler",
        target=self._HandleFlowProcessingRequestLoop,
        args=(handler,))
    self.flow_handler_thread.daemon = True
    self.flow_handler_thread.start()


Instruction:
Here you have a function in Python 3, explain what it does
def UnregisterFlowProcessingHandler(self, timeout=None):
    self.flow_handler_target = None

    if self.flow_handler_thread:
      self.flow_handler_stop = True
      self.flow_handler_thread.join(timeout)
      if self.flow_handler_thread.isAlive():
        raise RuntimeError("Flow processing handler did not join in time.")
      self.flow_handler_thread = None

Response: Unregisters any registered flow processing handler.


Instruction:
Explain what the following Python 3 code does
def WaitUntilNoFlowsToProcess(self, timeout=None):
    t = self.flow_handler_thread
    if not t:
      return

    start_time = time.time()
    while True:
      with self.lock:
        # If the thread is dead, or there are no requests
        # to be processed/being processed, we stop waiting
        # and return from the function.
        if (not t.isAlive() or
            (not self._GetFlowRequestsReadyForProcessing() and
             not self.flow_handler_num_being_processed)):
          return

      time.sleep(0.2)

      if timeout and time.time() - start_time > timeout:
        raise TimeOutWhileWaitingForFlowsToBeProcessedError(
            "Flow processing didn't finish in time.")

Response: Waits until all flows in the sequence are done processing flows.


Instruction:
Explain what the following Python 3 code does
def _HandleFlowProcessingRequestLoop(self, handler):
    while not self.flow_handler_stop:
      with self.lock:
        todo = self._GetFlowRequestsReadyForProcessing()
        for request in todo:
          self.flow_handler_num_being_processed += 1
          del self.flow_processing_requests[(request.client_id,
                                             request.flow_id)]

      for request in todo:
        handler(request)
        with self.lock:
          self.flow_handler_num_being_processed -= 1

      time.sleep(0.2)

Response: Handler thread for the FlowProcessingRequest queue.


Instruction:
Given the following Python 3 function, write the documentation
def WriteFlowResults(self, results):
    for r in results:
      dest = self.flow_results.setdefault((r.client_id, r.flow_id), [])
      to_write = r.Copy()
      to_write.timestamp = rdfvalue.RDFDatetime.Now()
      dest.append(to_write)

Response: Writes results for a given flow.


Instruction:
Can you generate the documentation for the following Python 3 function
def ReadFlowResults(self,
                      client_id,
                      flow_id,
                      offset,
                      count,
                      with_tag=None,
                      with_type=None,
                      with_substring=None):
    """Reads flow results of a given flow using given query options."""
    results = sorted(
        [x.Copy() for x in self.flow_results.get((client_id, flow_id), [])],
        key=lambda r: r.timestamp)

    # This is done in order to pass the tests that try to deserialize
    # value of an unrecognized type.
    for r in results:
      cls_name = compatibility.GetName(r.payload.__class__)
      if cls_name not in rdfvalue.RDFValue.classes:
        r.payload = rdf_objects.SerializedValueOfUnrecognizedType(
            type_name=cls_name, value=r.payload.SerializeToString())

    if with_tag is not None:
      results = [i for i in results if i.tag == with_tag]

    if with_type is not None:
      results = [
          i for i in results
          if compatibility.GetName(i.payload.__class__) == with_type
      ]

    if with_substring is not None:
      encoded_substring = with_substring.encode("utf8")
      results = [
          i for i in results
          if encoded_substring in i.payload.SerializeToString()
      ]

    return results[offset:offset + count]

Response: Reads the flow results of a given flow using given query options.


Instruction:
Write a Python 3 function that can
count the number of flow results of a given client and flow.

Response:
def CountFlowResults(self, client_id, flow_id, with_tag=None, with_type=None):
    """Counts flow results of a given flow using given query options."""
    return len(
        self.ReadFlowResults(
            client_id,
            flow_id,
            0,
            sys.maxsize,
            with_tag=with_tag,
            with_type=with_type))


Instruction:
Can you write a function in Python 3 where it
returns counts of flow results grouped by result type.

Response:
def CountFlowResultsByType(self, client_id, flow_id):
    """Returns counts of flow results grouped by result type."""
    result = collections.Counter()
    for hr in self.ReadFlowResults(client_id, flow_id, 0, sys.maxsize):
      key = compatibility.GetName(hr.payload.__class__)
      result[key] += 1

    return result


Instruction:
Write a Python 3 function for
writing log entries for a given flow.

Response:
def WriteFlowLogEntries(self, entries):
    """Writes flow output plugin log entries for a given flow."""
    flow_ids = [(e.client_id, e.flow_id) for e in entries]
    for f in flow_ids:
      if f not in self.flows:
        raise db.AtLeastOneUnknownFlowError(flow_ids)

    for e in entries:
      dest = self.flow_log_entries.setdefault((e.client_id, e.flow_id), [])
      to_write = e.Copy()
      to_write.timestamp = rdfvalue.RDFDatetime.Now()
      dest.append(to_write)


Instruction:
Create a Python 3 function for
reading flow log entries of a given flow.

Response:
def ReadFlowLogEntries(self,
                         client_id,
                         flow_id,
                         offset,
                         count,
                         with_substring=None):
    """Reads flow log entries of a given flow using given query options."""
    entries = sorted(
        self.flow_log_entries.get((client_id, flow_id), []),
        key=lambda e: e.timestamp)

    if with_substring is not None:
      entries = [i for i in entries if with_substring in i.message]

    return entries[offset:offset + count]


Instruction:
Given the following Python 3 function, write the documentation
def CountFlowLogEntries(self, client_id, flow_id):
    return len(self.ReadFlowLogEntries(client_id, flow_id, 0, sys.maxsize))

Response: Returns the number of flow log entries of a given flow.


Instruction:
Write a Python 3 function that can
read flow output plugin log entries.

Response:
def ReadFlowOutputPluginLogEntries(self,
                                     client_id,
                                     flow_id,
                                     output_plugin_id,
                                     offset,
                                     count,
                                     with_type=None):
    """Reads flow output plugin log entries."""
    entries = sorted(
        self.flow_output_plugin_log_entries.get((client_id, flow_id), []),
        key=lambda e: e.timestamp)

    entries = [e for e in entries if e.output_plugin_id == output_plugin_id]

    if with_type is not None:
      entries = [e for e in entries if e.log_entry_type == with_type]

    return entries[offset:offset + count]


Instruction:
Explain what the following Python 3 code does
def CountFlowOutputPluginLogEntries(self,
                                      client_id,
                                      flow_id,
                                      output_plugin_id,
                                      with_type=None):
    """Returns number of flow output plugin log entries of a given flow."""

    return len(
        self.ReadFlowOutputPluginLogEntries(
            client_id,
            flow_id,
            output_plugin_id,
            0,
            sys.maxsize,
            with_type=with_type))

Response: Returns the number of flow output plugin log entries of a given flow.


Instruction:
How would you code a function in Python 3 to
write signed binary references.

Response:
def WriteSignedBinaryReferences(self, binary_id,
                                  references):
    """See db.Database."""
    self.signed_binary_references[_SignedBinaryKeyFromID(binary_id)] = (
        references.Copy(), rdfvalue.RDFDatetime.Now())


Instruction:
Write a Python 3 function that can
read the signed binary references.

Response:
def ReadSignedBinaryReferences(
      self, binary_id
  ):
    """See db.Database."""
    binary_key = _SignedBinaryKeyFromID(binary_id)
    try:
      references, timestamp = self.signed_binary_references[binary_key]
    except KeyError:
      raise db.UnknownSignedBinaryError(binary_id)
    return references.Copy(), timestamp.Copy()