Instruction:
Create a Python 3 function to
create a CheckAccessHelper for controlling query access.

Response:
def _CreateQueryAccessHelper(self):
    """Creates a CheckAccessHelper for controlling query access.

    This function and _CreateReadAccessHelper essentially define GRR's ACL
    policy. Please refer to these 2 functions to either review or modify
    GRR's ACLs.

    Query access gives you the ability to find objects in the tree without
    knowing their URN, using ListChildren.  If you grant query access,
    you will also need read access.

    Returns:
      CheckAccessHelper for controlling query access.
    """
    h = CheckAccessHelper("query")

    # User is allowed to do anything in their home dir.
    h.Allow("aff4:/users/*", self._IsHomeDir)

    # Querying is allowed for aff4:/cron/*, as GUI renders list of current
    # cron jobs. Also, links to flows executed by every cron job are stored
    # below cron job object,
    h.Allow("aff4:/cron")
    h.Allow("aff4:/cron/*")

    # Hunts and data collected by hunts are publicly available and thus can
    # be queried.
    h.Allow("aff4:/hunts")
    h.Allow("aff4:/hunts/*")

    # ACLs have to be queried to search for proper approvals. As a consequence,
    # any user can query for all approval objects and get a list of all
    # possible reasons. At the moment we assume that it's ok.
    h.Allow("aff4:/ACL")
    h.Allow("aff4:/ACL/*")

    # Querying contents of the client is allowed, as we have to browse VFS
    # filesystem.
    h.Allow(self.CLIENT_URN_PATTERN)
    h.Allow(self.CLIENT_URN_PATTERN + "/*")

    # Namespace for indexes. Client index is stored there and users need to
    # query the index for searching clients.
    h.Allow("aff4:/index")
    h.Allow("aff4:/index/*")

    # Configuration namespace used for reading drivers, python hacks etc. The
    # GUI needs to be able to list these paths.
    h.Allow("aff4:/config")
    h.Allow("aff4:/config/*")

    # We do not allow querying for all flows with empty client id, but we do
    # allow querying for particular flow's children. This is needed, in
    # particular, for cron jobs, whose flows end up in aff4:/flows and we
    # have to query them for children flows.
    # NOTE: Allow(aff4:/flows/*) does not allow querying of aff4:/flows
    #       children. But, on the other hand, it will allow querying for
    #       anything under aff4:/flows/W:SOMEFLOW, for example.
    h.Allow("aff4:/flows/*")

    # Querying the index of filestore objects is allowed since it reveals the
    # clients which have this file.
    h.Allow("aff4:/files/hash/generic/sha256/" + "[a-z0-9]" * 64)

    # Allow everyone to query the artifact store.
    h.Allow("aff4:/artifact_store")
    h.Allow("aff4:/artifact_store/*")

    # Users are allowed to query the artifact store.
    h.Allow("aff4:/artifact_store")
    h.Allow("aff4:/artifact_store/*")

    # Namespace for audit data. Required to display usage statistics.
    h.Allow("aff4:/audit/logs")
    h.Allow("aff4:/audit/logs/*")

    return h


Instruction:
How would you explain what the following Python 3 function does
def CheckDataStoreAccess(self, token, subjects, requested_access="r"):
    if any(not x for x in subjects):
      raise ValueError("Subjects list can't contain empty URNs.")
    subjects = list(map(rdfvalue.RDFURN, subjects))

    return (ValidateAccessAndSubjects(requested_access, subjects) and
            ValidateToken(token, subjects) and
            (token.supervisor or
             self._CheckAccessWithHelpers(token, subjects, requested_access)))

Response: Checks if token and requested access are valid.


Instruction:
Create a Python 3 function for
streaming chunks of a given file.

Response:
def StreamFile(self, filedesc, offset=0, amount=None):
    """Streams chunks of a given file starting at given offset.

    Args:
      filedesc: A `file` object to stream.
      offset: An integer offset at which the file stream should start on.
      amount: An upper bound on number of bytes to read.

    Returns:
      Generator over `Chunk` instances.
    """
    reader = FileReader(filedesc, offset=offset)
    return self.Stream(reader, amount=amount)


Instruction:
Here you have a function in Python 3, explain what it does
def StreamFilePath(self, filepath, offset=0, amount=None):
    with open(filepath, "rb") as filedesc:
      for chunk in self.StreamFile(filedesc, offset=offset, amount=amount):
        yield chunk

Response: Streams chunks of a file located at given path starting at given offset.


Instruction:
Implement a Python 3 function for
streaming chunks of memory of a given process starting at given offset.

Response:
def StreamMemory(self, process, offset=0, amount=None):
    """Streams chunks of memory of a given process starting at given offset.

    Args:
      process: A platform-specific `Process` instance.
      offset: An integer offset at which the memory stream should start on.
      amount: An upper bound on number of bytes to read.

    Returns:
      Generator over `Chunk` instances.
    """
    reader = MemoryReader(process, offset=offset)
    return self.Stream(reader, amount=amount)


Instruction:
How would you explain what the following Python 3 function does
def Stream(self, reader, amount=None):
    if amount is None:
      amount = float("inf")

    data = reader.Read(min(self.chunk_size, amount))
    if not data:
      return

    amount -= len(data)
    offset = reader.offset - len(data)
    yield Chunk(offset=offset, data=data)

    while amount > 0:
      # We need `len(data)` here because overlap size can be 0.
      overlap = data[len(data) - self.overlap_size:]

      new = reader.Read(min(self.chunk_size - self.overlap_size, amount))
      if not new:
        return

      data = overlap + new

      amount -= len(new)
      offset = reader.offset - len(data)
      yield Chunk(offset=offset, data=data, overlap=len(overlap))

Response: Streams chunks of a given file.


Instruction:
Make a summary of the following Python 3 code
def Scan(self, matcher):

    position = 0
    while True:
      span = matcher.Match(self.data, position)
      if span is None:
        return

      # We are not interested in hits within overlap-only zone. We continue the
      # search just after the previous match starts because of situations where
      # there is a match beginning before the end of the overlap-only zone match
      # and ending after the overlap zone.
      if span.end <= self.overlap:
        position = span.begin + 1
        continue

      # Since we do not care about overlapping matches we resume our search
      # at the end of the previous match.
      position = span.end
      yield span

Response: Yields spans occurrences of a given pattern within a chunk.


Instruction:
Create a Python 3 function to
list all the filesystems mounted on the system.

Response:
def GetMountpoints(data=None):
  """List all the filesystems mounted on the system."""
  expiry = 60  # 1 min

  insert_time = MOUNTPOINT_CACHE[0]
  if insert_time + expiry > time.time():
    return MOUNTPOINT_CACHE[1]

  devices = {}

  # Check all the mounted filesystems.
  if data is None:
    data = "\n".join(
        [open(x, "rb").read() for x in ["/proc/mounts", "/etc/mtab"]])

  for line in data.splitlines():
    try:
      device, mnt_point, fs_type, _ = line.split(" ", 3)
      mnt_point = os.path.normpath(mnt_point)

      # What if several devices are mounted on the same mount point?
      devices[mnt_point] = (device, fs_type)
    except ValueError:
      pass

  MOUNTPOINT_CACHE[0] = time.time()
  MOUNTPOINT_CACHE[1] = devices

  return devices


Instruction:
Can you write a function in Python 3 where it
executes a command on the GRR client.

Response:
def Execute(cmd,
            args,
            time_limit=-1,
            bypass_whitelist=False,
            daemon=False,
            use_client_context=False,
            cwd=None):
  """Executes commands on the client.

  This function is the only place where commands will be executed
  by the GRR client. This makes sure that all issued commands are compared to a
  white list and no malicious commands are issued on the client machine.

  Args:
    cmd: The command to be executed.
    args: List of arguments.
    time_limit: Time in seconds the process is allowed to run.
    bypass_whitelist: Allow execution of things that are not in the whitelist.
      Note that this should only ever be called on a binary that passes the
      VerifySignedBlob check.
    daemon: Start the new process in the background.
    use_client_context: Run this script in the client's context. Defaults to
      system context.
    cwd: Current working directory for the command.

  Returns:
    A tuple of stdout, stderr, return value and time taken.
  """
  if not bypass_whitelist and not IsExecutionWhitelisted(cmd, args):
    # Whitelist doesn't contain this cmd/arg pair
    logging.info("Execution disallowed by whitelist: %s %s.", cmd,
                 " ".join(args))
    return (b"", b"Execution disallowed by whitelist.", -1, -1)

  if daemon:
    pid = os.fork()
    if pid == 0:
      # This is the child, it will run the daemon process. We call os.setsid
      # here to become the session leader of this new session and the process
      # group leader of the new process group so we don't get killed when the
      # main process exits.
      try:
        os.setsid()
      except OSError:
        # This only works if the process is running as root.
        pass
      _Execute(
          cmd, args, time_limit, use_client_context=use_client_context, cwd=cwd)
      os._exit(0)  # pylint: disable=protected-access
  else:
    return _Execute(
        cmd, args, time_limit, use_client_context=use_client_context, cwd=cwd)


Instruction:
Explain what the following Python 3 code does
def _Execute(cmd, args, time_limit=-1, use_client_context=False, cwd=None):
  run = [cmd]
  run.extend(args)
  env = os.environ.copy()
  if use_client_context:
    env.pop("LD_LIBRARY_PATH", None)
    env.pop("PYTHON_PATH", None)
    context = "client"
  else:
    context = "system"
  logging.info("Executing %s in %s context.", " ".join(run), context)
  p = subprocess.Popen(
      run,
      stdin=subprocess.PIPE,
      stdout=subprocess.PIPE,
      stderr=subprocess.PIPE,
      env=env,
      cwd=cwd)

  alarm = None
  if time_limit > 0:
    alarm = threading.Timer(time_limit, HandleAlarm, (p,))
    alarm.setDaemon(True)
    alarm.start()

  stdout, stderr, exit_status = b"", b"", -1
  start_time = time.time()
  try:
    stdout, stderr = p.communicate()
    exit_status = p.returncode
  except IOError:
    # If we end up here, the time limit was exceeded
    pass
  finally:
    if alarm:
      alarm.cancel()
      alarm.join()

  return (stdout, stderr, exit_status, time.time() - start_time)

Response: Executes cmd with args.


Instruction:
How would you code a function in Python 3 to
check if a binary and args are whitelisted.

Response:
def IsExecutionWhitelisted(cmd, args):
  """Check if a binary and args is whitelisted.

  Args:
    cmd: Canonical path to the binary.
    args: List of arguments to be passed to the binary.

  Returns:
    Bool, True if it is whitelisted.

  These whitelists could also go in the platform specific client files
  client_utils_<platform>.py. We chose to leave them here instead of putting
  them in global arrays to discourage people coding other modules from adding
  new commands to the whitelist before running them.
  The idea is to have a single place that lists every command we can run during
  normal operation (obviously doesn't catch the special cases where we bypass
  the list).
  A deployment-specific list is also checked (see local/binary_whitelist.py).
  """
  if platform.system() == "Windows":
    whitelist = [
        ("arp.exe", ["-a"]),
        ("driverquery.exe", ["/v"]),
        ("ipconfig.exe", ["/all"]),
        ("netsh.exe", ["advfirewall", "firewall", "show", "rule", "name=all"]),
        ("netsh.exe",
         ["advfirewall", "monitor", "show", "firewall", "rule", "name=all"]),
        ("tasklist.exe", ["/SVC"]),
        ("tasklist.exe", ["/v"]),
    ]
  elif platform.system() == "Linux":
    whitelist = [
        ("/bin/df", []),
        ("/bin/echo", ["1"]),
        ("/bin/rpm", ["-qa"]),
        ("/bin/sleep", ["10"]),
        ("/sbin/auditctl", ["-l"]),
        ("/sbin/ifconfig", ["-a"]),
        ("/sbin/iptables", ["-L", "-n", "-v"]),
        ("/sbin/lsmod", []),
        ("/usr/bin/dpkg", ["--list"]),
        ("/usr/bin/last", []),
        ("/usr/bin/yum", ["list", "installed", "-q"]),
        ("/usr/bin/yum", ["repolist", "-v", "-q"]),
        ("/usr/bin/who", []),
        ("/usr/sbin/arp", ["-a"]),
        ("/usr/sbin/dmidecode", ["-q"]),
        ("/usr/sbin/sshd", ["-T"]),
    ]
  elif platform.system() == "Darwin":
    whitelist = [
        ("/bin/echo", ["1"]),
        ("/bin/launchctl", ["unload", config.CONFIG["Client.plist_path"]]),
        ("/usr/bin/hdiutil", ["info"]),
        ("/usr/bin/last", []),
        ("/usr/bin/who", []),
        ("/usr/sbin/arp", ["-a"]),
        ("/usr/sbin/kextstat", []),
        ("/usr/sbin/system_profiler", ["-xml", "SPHardwareDataType"]),
        ("/usr/libexec/firmwarecheckers/ethcheck/ethcheck", ["--show-hashes"]),
    ]
  else:
    whitelist = []

  for (allowed_cmd, allowed_args) in whitelist:
    if cmd == allowed_cmd and args == allowed_args:
      return True

  # Check if this is whitelisted locally.
  if binary_whitelist.IsExecutionWhitelisted(cmd, args):
    return True

  return False


Instruction:
Can you generate a brief explanation for the following Python 3 code
def HashFilePath(self, path, byte_count):
    with open(path, "rb") as fd:
      self.HashFile(fd, byte_count)

Response: Updates underlying hashers with file on a given path.


Instruction:
Here you have a function in Python 3, explain what it does
def HashFile(self, fd, byte_count):
    while byte_count > 0:
      buf_size = min(byte_count, constants.CLIENT_MAX_BUFFER_SIZE)
      buf = fd.read(buf_size)
      if not buf:
        break

      self.HashBuffer(buf)
      byte_count -= buf_size

Response: Updates underlying hashers with a given file object.


Instruction:
Given the following Python 3 function, write the documentation
def HashBuffer(self, buf):
    for hasher in itervalues(self._hashers):
      hasher.update(buf)
      if self._progress:
        self._progress()

    self._bytes_read += len(buf)

Response: Updates the underlying hashers with a given byte buffer.


Instruction:
How would you implement a function in Python 3 that
returns a Hash object with appropriate fields filled - in.

Response:
def GetHashObject(self):
    """Returns a `Hash` object with appropriate fields filled-in."""
    hash_object = rdf_crypto.Hash()
    hash_object.num_bytes = self._bytes_read
    for algorithm in self._hashers:
      setattr(hash_object, algorithm, self._hashers[algorithm].digest())
    return hash_object


Instruction:
Make a summary of the following Python 3 code
def GetAllClientLabels(token, include_catchall=False):
  labels_index = aff4.FACTORY.Create(
      standard.LabelSet.CLIENT_LABELS_URN,
      standard.LabelSet,
      mode="r",
      token=token)
  labels = set(labels_index.ListLabels())
  if include_catchall:
    labels.add(ALL_CLIENTS_LABEL)
  return labels

Response: Returns the set of all label names applied to all clients.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def age(self):
    # TODO(user) move up to AFF4Object after some analysis of how .age is
    # used in the codebase.
    aff4_type = self.Get(self.Schema.TYPE)

    if aff4_type:
      return aff4_type.age
    else:
      # If there is no type attribute yet, we have only just been created and
      # not flushed yet, so just set timestamp to now.
      return rdfvalue.RDFDatetime.Now()

Response: Returns the age of the object.


Instruction:
Explain what the following Python 3 code does
def GetSummary(self):
    self.max_age = 0
    summary = rdf_client.ClientSummary(client_id=self.urn)
    summary.system_info.system = self.Get(self.Schema.SYSTEM)
    summary.system_info.release = self.Get(self.Schema.OS_RELEASE)
    summary.system_info.version = str(self.Get(self.Schema.OS_VERSION, ""))
    summary.system_info.kernel = self.Get(self.Schema.KERNEL)
    summary.system_info.fqdn = self.Get(self.Schema.FQDN)
    summary.system_info.machine = self.Get(self.Schema.ARCH)
    summary.system_info.install_date = self.Get(self.Schema.INSTALL_DATE)
    kb = self.Get(self.Schema.KNOWLEDGE_BASE)
    if kb:
      summary.users = kb.users
    summary.interfaces = self.Get(self.Schema.INTERFACES)
    summary.client_info = self.Get(self.Schema.CLIENT_INFO)
    hwi = self.Get(self.Schema.HARDWARE_INFO)
    if hwi:
      summary.serial_number = hwi.serial_number
      summary.system_manufacturer = hwi.system_manufacturer
      summary.system_uuid = hwi.system_uuid
    summary.timestamp = self.age
    cloud_instance = self.Get(self.Schema.CLOUD_INSTANCE)
    if cloud_instance:
      summary.cloud_type = cloud_instance.cloud_type
      if cloud_instance.cloud_type == "GOOGLE":
        summary.cloud_instance_id = cloud_instance.google.unique_id
      elif cloud_instance.cloud_type == "AMAZON":
        summary.cloud_instance_id = cloud_instance.amazon.instance_id
      else:
        raise ValueError("Bad cloud type: %s" % cloud_instance.cloud_type)

    return summary

Response: Retrieves a client summary object.


Instruction:
Make a summary of the following Python 3 code
def Start(self):
    self.Init()
    fd = aff4.FACTORY.Open(self.args.vfs_file_urn, mode="rw", token=self.token)

    # Account for implicit directories.
    if fd.Get(fd.Schema.TYPE) is None:
      fd = fd.Upgrade(standard.VFSDirectory)

    self.state.get_file_flow_urn = fd.Update(attribute=self.args.attribute)

Response: Calls the Update method of a given VFSFile or VFSDirectory object.


Instruction:
Can you create a Python 3 function that
updates an attribute from the client.

Response:
def Update(self, attribute=None):
    """Update an attribute from the client."""
    # List the directory on the client
    currently_running = self.Get(self.Schema.CONTENT_LOCK)

    # Is this flow still active?
    if currently_running:
      flow_obj = aff4.FACTORY.Open(currently_running, token=self.token)
      if flow_obj and flow_obj.GetRunner().IsRunning():
        return

    # The client_id is the first element of the URN
    client_id = self.urn.Path().split("/", 2)[1]

    # Get the pathspec for this object
    pathspec = self.Get(self.Schema.STAT).pathspec
    flow_urn = flow.StartAFF4Flow(
        client_id=client_id,
        # TODO(user): dependency loop between aff4_grr.py and transfer.py
        # flow_name=transfer.MultiGetFile.__name__,
        flow_name="MultiGetFile",
        token=self.token,
        pathspecs=[pathspec])
    self.Set(self.Schema.CONTENT_LOCK(flow_urn))
    self.Close()

    return flow_urn


Instruction:
Explain what the following Python 3 code does
def ExpireRules(self):
    rules = self.Get(self.Schema.RULES)
    new_rules = self.Schema.RULES()
    now = time.time() * 1e6
    expired_session_ids = set()

    for rule in rules:
      if rule.expires > now:
        new_rules.Append(rule)
      else:
        for action in rule.actions:
          if action.hunt_id:
            expired_session_ids.add(action.hunt_id)

    if expired_session_ids:
      with data_store.DB.GetMutationPool() as pool:
        # Notify the worker to mark this hunt as terminated.
        manager = queue_manager.QueueManager(token=self.token)
        manager.MultiNotifyQueue([
            rdf_flows.GrrNotification(session_id=session_id)
            for session_id in expired_session_ids
        ],
                                 mutation_pool=pool)

    if len(new_rules) < len(rules):
      self.Set(self.Schema.RULES, new_rules)
      self.Flush()

Response: Removes any rules with an expiration date in the past.


Instruction:
Create a Python 3 function to
return True if hunt s task was assigned to this client before.

Response:
def _CheckIfHuntTaskWasAssigned(self, client_id, hunt_id):
    """Will return True if hunt's task was assigned to this client before."""
    client_urn = rdfvalue.RDFURN(client_id)
    for _ in aff4.FACTORY.Stat([
        client_urn.Add("flows/%s:hunt" % rdfvalue.RDFURN(hunt_id).Basename())
    ]):
      return True

    return False


Instruction:
Can you tell what is the following Python 3 function doing
def _RunActions(self, rule, client_id):
    actions_count = 0

    for action in rule.actions:
      try:
        # Say this flow came from the foreman.
        token = self.token.Copy()
        token.username = "Foreman"

        if action.HasField("hunt_id"):
          if self._CheckIfHuntTaskWasAssigned(client_id, action.hunt_id):
            logging.info(
                "Foreman: ignoring hunt %s on client %s: was started "
                "here before", client_id, action.hunt_id)
          else:
            logging.info("Foreman: Starting hunt %s on client %s.",
                         action.hunt_id, client_id)

            flow_cls = registry.AFF4FlowRegistry.FlowClassByName(
                action.hunt_name)
            flow_cls.StartClients(action.hunt_id, [client_id])
            actions_count += 1
        else:
          flow.StartAFF4Flow(
              client_id=client_id,
              flow_name=action.flow_name,
              token=token,
              **action.argv.ToDict())
          actions_count += 1
      # There could be all kinds of errors we don't know about when starting the
      # flow/hunt so we catch everything here.
      except Exception as e:  # pylint: disable=broad-except
        logging.exception("Failure running foreman action on client %s: %s",
                          action.hunt_id, e)

    return actions_count

Response: Runs all the actions specified in the rule.


Instruction:
Create a Python 3 function to
assign tasks to a client.

Response:
def AssignTasksToClient(self, client_id):
    """Examines our rules and starts up flows based on the client.

    Args:
      client_id: Client id of the client for tasks to be assigned.

    Returns:
      Number of assigned tasks.
    """
    rules = self.Get(self.Schema.RULES)
    if not rules:
      return 0

    if data_store.RelationalDBEnabled():
      last_foreman_run = self._GetLastForemanRunTimeRelational(client_id)
    else:
      last_foreman_run = self._GetLastForemanRunTime(client_id)

    latest_rule = max(rule.created for rule in rules)

    if latest_rule <= last_foreman_run:
      return 0

    # Update the latest checked rule on the client.
    if data_store.RelationalDBEnabled():
      try:
        self._SetLastForemanRunTimeRelational(client_id, latest_rule)
      except db.UnknownClientError:
        pass

    # If the relational db is used for reads, we don't have to update the
    # aff4 object.
    if not data_store.RelationalDBEnabled():
      self._SetLastForemanRunTime(client_id, latest_rule)

    relevant_rules = []
    expired_rules = False

    now = time.time() * 1e6

    for rule in rules:
      if rule.expires < now:
        expired_rules = True
        continue
      if rule.created <= int(last_foreman_run):
        continue

      relevant_rules.append(rule)

    if data_store.RelationalDBEnabled():
      client_data = data_store.REL_DB.ReadClientFullInfo(client_id)
      if client_data is None:
        return
    else:
      client_data = aff4.FACTORY.Open(client_id, mode="rw", token=self.token)

    actions_count = 0
    for rule in relevant_rules:
      if self._EvaluateRules(rule, client_data):
        actions_count += self._RunActions(rule, client_id)

    if expired_rules:
      self.ExpireRules()

    return actions_count


Instruction:
Can you create a Python 3 function that
opens the delegate object.

Response:
def Initialize(self):
    """Open the delegate object."""
    if "r" in self.mode:
      delegate = self.Get(self.Schema.DELEGATE)
      if delegate:
        self.delegate = aff4.FACTORY.Open(
            delegate, mode=self.mode, token=self.token, age=self.age_policy)


Instruction:
Can you write a function in Python 3 where it
retrieves the relevant blob from the AFF4 data store or cache.

Response:
def _GetChunkForReading(self, chunk):
    """Retrieve the relevant blob from the AFF4 data store or cache."""
    offset = chunk * self._HASH_SIZE
    self.index.seek(offset)

    chunk_name = self.index.read(self._HASH_SIZE)

    try:
      return self.chunk_cache.Get(chunk_name)
    except KeyError:
      pass

    # We don't have this chunk already cached. The most common read
    # access pattern is contiguous reading so since we have to go to
    # the data store already, we read ahead to reduce round trips.
    self.index.seek(offset)
    readahead = []

    for _ in range(self._READAHEAD):
      name = self.index.read(self._HASH_SIZE)
      if name and name not in self.chunk_cache:
        readahead.append(rdf_objects.BlobID.FromBytes(name))

    self._ReadChunks(readahead)
    try:
      return self.chunk_cache.Get(chunk_name)
    except KeyError:
      raise aff4.ChunkNotFoundError("Cannot open chunk %s" % chunk)


Instruction:
How would you code a function in Python 3 to
append content to the blob image.

Response:
def AppendContent(self, src_fd):
    """Create new blob hashes and append to BlobImage.

    We don't support writing at arbitrary file offsets, but this method provides
    a convenient way to add blobs for a new file, or append content to an
    existing one.

    Args:
      src_fd: source file handle open for read

    Raises:
      IOError: if blob has already been finalized.
    """
    while 1:
      blob = src_fd.read(self.chunksize)
      if not blob:
        break

      blob_id = data_store.BLOBS.WriteBlobWithUnknownHash(blob)
      self.AddBlob(blob_id, len(blob))

    self.Flush()


Instruction:
Implement a Python 3 function for
adding a blob to this blob image using its hash.

Response:
def AddBlob(self, blob_id, length):
    """Add another blob to this image using its hash.

    Once a blob is added that is smaller than the chunksize we finalize the
    file, since handling adding more blobs makes the code much more complex.

    Args:
      blob_id: rdf_objects.BlobID object.
      length: int length of blob

    Raises:
      IOError: if blob has been finalized.
    """
    if self.finalized and length > 0:
      raise IOError("Can't add blobs to finalized BlobImage")

    self.content_dirty = True
    self.index.seek(0, 2)
    self.index.write(blob_id.AsBytes())
    self.size += length

    if length < self.chunksize:
      self.finalized = True


Instruction:
Given the following Python 3 function, write the documentation
def NewFromContent(cls,
                     content,
                     urn,
                     chunk_size=1024,
                     token=None,
                     private_key=None,
                     public_key=None):
    """Alternate constructor for GRRSignedBlob.

    Creates a GRRSignedBlob from a content string by chunking it and signing
    each chunk.

    Args:
      content: The data to stored in the GRRSignedBlob.
      urn: The AFF4 URN to create.

      chunk_size: Data will be chunked into this size (each chunk is
        individually signed.
      token: The ACL Token.
      private_key: An rdf_crypto.RSAPrivateKey() instance.
      public_key: An rdf_crypto.RSAPublicKey() instance.

    Returns:
      the URN of the new object written.
    """
    aff4.FACTORY.Delete(urn, token=token)
    with data_store.DB.GetMutationPool() as pool:
      with aff4.FACTORY.Create(
          urn, cls, mode="w", mutation_pool=pool, token=token) as fd:
        for start_of_chunk in range(0, len(content), chunk_size):
          chunk = content[start_of_chunk:start_of_chunk + chunk_size]
          blob_rdf = rdf_crypto.SignedBlob()
          blob_rdf.Sign(chunk, private_key, public_key)
          fd.Add(blob_rdf, mutation_pool=pool)

    return urn

Response: Alternate constructor for GRRSignedBlob.


Instruction:
Can you tell what is the following Python 3 function doing
def Substitute(self, pattern):
    if isinstance(pattern, bytes):
      substs = [re.escape(subst.encode("ascii")) for subst in self._substs]
      regex = re.compile(b"|".join(substs))

      def Replacement(match):
        key = match.group(0).decode("ascii")
        return self._substs[key].encode("utf-8")

    elif isinstance(pattern, Text):
      substs = [re.escape(subst) for subst in self._substs]
      regex = re.compile("|".join(substs))

      def Replacement(match):
        key = match.group(0)
        return self._substs[key]

    else:
      raise TypeError("Unexpected pattern type '{}'".format(type(pattern)))

    if not substs:
      return pattern
    else:
      return regex.sub(Replacement, pattern)

Response: Formats given pattern with this substitution environment.


Instruction:
Can you implement a function in Python 3 that
associates a value with a given variable.

Response:
def BindVar(self, var_id, value):
    """Associates a value with given variable.

    This can be called multiple times to associate multiple values.

    Args:
      var_id: A variable id to bind the values to.
      value: A value to bind to the specified variable.

    Raises:
      KeyError: If given variable is not specified in the pattern.
    """
    if var_id not in self._vars:
      raise KeyError(var_id)

    self._var_bindings[var_id].append(value)


Instruction:
How would you code a function in Python 3 to
associate given values with given scope.

Response:
def BindScope(self, scope_id, values):
    """Associates given values with given scope.

    This can be called multiple times to associate multiple values.

    Args:
      scope_id: A scope id to bind the values to.
      values: A mapping from scope variable ids to values to bind in scope.

    Raises:
      KeyError: If given scope or scope variable is not specified in the
        pattern.
    """
    if scope_id not in self._scopes:
      raise KeyError(scope_id)

    keys = set(iterkeys(values))
    if keys != self._scopes[scope_id]:
      raise KeyError(keys ^ self._scopes[scope_id])

    self._scope_bindings[scope_id].append(values)


Instruction:
Explain what the following Python 3 code does
def Interpolate(self):
    for var_config in collection.DictProduct(self._var_bindings):
      for scope_config in collection.DictProduct(self._scope_bindings):
        subst = Substitution(var_config=var_config, scope_config=scope_config)
        yield subst.Substitute(self._pattern)

Response:
Interpolates the pattern.

    Yields:
      All possible interpolation results.


Instruction:
Write a Python 3 script to
register a report plugin for use in the GRR UI.

Response:
def RegisterPlugin(self, report_plugin_cls):
    """Registers a report plugin for use in the GRR UI."""

    name = report_plugin_cls.__name__
    if name in self.plugins:
      raise RuntimeError("Can't register two report plugins with the same "
                         "name. In particular, can't register the same "
                         "report plugin twice: %r" % name)

    self.plugins[name] = report_plugin_cls


Instruction:
Explain what the following Python 3 code does
def MakeExecutableTemplate(self, output_file=None):
    super(DarwinClientBuilder,
          self).MakeExecutableTemplate(output_file=output_file)
    self.SetBuildVars()
    self.MakeBuildDirectory()
    self.BuildWithPyInstaller()
    self.CopyMissingModules()
    self.BuildInstallerPkg(output_file)
    self.MakeZip(output_file, self.template_file)

Response: Create the executable template.


Instruction:
Create a Python 3 function for
adding a zip to the end of the .xar containing build.yaml. The build.yaml is already inside the .xar file, but we can't easily open this on linux. To make repacking easier we add a zip to the end of the .xar and add in the build.yaml. The repack step will then look at the build.yaml and insert the config.yaml. We end up storing the build.yaml twice but it is tiny, so this doesn't matter. Args: xar_file: the name of the xar file. output_file: the name of the output ZIP archive.

Response:
def MakeZip(self, xar_file, output_file):
    """Add a zip to the end of the .xar containing build.yaml.

    The build.yaml is already inside the .xar file, but we can't easily open
    this on linux. To make repacking easier we add a zip to the end of the .xar
    and add in the build.yaml. The repack step will then look at the build.yaml
    and insert the config.yaml. We end up storing the build.yaml twice but it is
    tiny, so this doesn't matter.

    Args:
      xar_file: the name of the xar file.
      output_file: the name of the output ZIP archive.
    """
    logging.info("Generating zip template file at %s", output_file)
    with zipfile.ZipFile(output_file, mode="a") as zf:
      # Get the build yaml
      # TODO(hanuszczak): YAML, consider using `StringIO` instead.
      build_yaml = io.BytesIO()
      self.WriteBuildYaml(build_yaml)
      build_yaml.seek(0)
      zf.writestr("build.yaml", build_yaml.read())


Instruction:
Write a Python 3 script to
build a package using PackageMaker.

Response:
def BuildInstallerPkg(self, output_file):
    """Builds a package (.pkg) using PackageMaker."""
    self.CreateInstallDirs()
    self.InterpolateFiles()
    self.RenameGRRPyinstallerBinaries()
    self.SignGRRPyinstallerBinaries()
    self.WriteClientConfig()
    self.Set755Permissions()
    self.RunPkgBuild()
    self.RunProductBuild()
    self.RenamePkgToTemplate(output_file)


Instruction:
How would you code a function in Python 3 to
return a path - like String of client_path with optional prefix.

Response:
def _ClientPathToString(client_path, prefix=""):
  """Returns a path-like String of client_path with optional prefix."""
  return os.path.join(prefix, client_path.client_id, client_path.vfs_path)


Instruction:
Create a Python 3 function to
generate a MANIFEST file in the archive.

Response:
def _GenerateDescription(self):
    """Generates description into a MANIFEST file in the archive."""

    manifest = {
        "description": self.description,
        "processed_files": len(self.processed_files),
        "archived_files": len(self.archived_files),
        "ignored_files": len(self.ignored_files),
        "failed_files": len(self.failed_files)
    }
    if self.ignored_files:
      manifest["ignored_files_list"] = [
          _ClientPathToString(cp, prefix="aff4:") for cp in self.ignored_files
      ]
    if self.failed_files:
      manifest["failed_files_list"] = [
          _ClientPathToString(cp, prefix="aff4:") for cp in self.failed_files
      ]

    manifest_fd = io.BytesIO()
    if self.total_files != len(self.archived_files):
      manifest_fd.write(self.FILES_SKIPPED_WARNING)
    manifest_fd.write(yaml.Dump(manifest).encode("utf-8"))

    manifest_fd.seek(0)
    st = os.stat_result(
        (0o644, 0, 0, 0, 0, 0, len(manifest_fd.getvalue()), 0, 0, 0))

    for chunk in self.archive_generator.WriteFromFD(
        manifest_fd, os.path.join(self.prefix, "MANIFEST"), st=st):
      yield chunk


Instruction:
Can you tell what is the following Python 3 function doing
def _GenerateClientInfo(self, client_id, client_fd):
    summary_dict = client_fd.ToPrimitiveDict(stringify_leaf_fields=True)
    summary = yaml.Dump(summary_dict).encode("utf-8")

    client_info_path = os.path.join(self.prefix, client_id, "client_info.yaml")
    st = os.stat_result((0o644, 0, 0, 0, 0, 0, len(summary), 0, 0, 0))
    yield self.archive_generator.WriteFileHeader(client_info_path, st=st)
    yield self.archive_generator.WriteFileChunk(summary)
    yield self.archive_generator.WriteFileFooter()

Response: Generates the client info file.


Instruction:
Implement a function in Python 3 to
generate an AFF4 archive from a given collection.

Response:
def Generate(self, items, token=None):
    """Generates archive from a given collection.

    Iterates the collection and generates an archive by yielding contents
    of every referenced AFF4Stream.

    Args:
      items: Iterable of rdf_client_fs.StatEntry objects
      token: User's ACLToken.

    Yields:
      Binary chunks comprising the generated archive.
    """

    del token  # unused, to be removed with AFF4 code

    client_ids = set()
    for item_batch in collection.Batch(items, self.BATCH_SIZE):

      client_paths = set()
      for item in item_batch:
        try:
          client_path = flow_export.CollectionItemToClientPath(
              item, self.client_id)
        except flow_export.ItemNotExportableError:
          continue

        if not self.predicate(client_path):
          self.ignored_files.add(client_path)
          self.processed_files.add(client_path)
          continue

        client_ids.add(client_path.client_id)
        client_paths.add(client_path)

      for chunk in file_store.StreamFilesChunks(client_paths):
        self.processed_files.add(chunk.client_path)
        for output in self._WriteFileChunk(chunk=chunk):
          yield output

      self.processed_files |= client_paths - (
          self.ignored_files | self.archived_files)

    if client_ids:
      for client_id, client_info in iteritems(
          data_store.REL_DB.MultiReadClientFullInfo(client_ids)):
        client = api_client.ApiClient().InitFromClientInfo(client_info)
        for chunk in self._GenerateClientInfo(client_id, client):
          yield chunk

    for chunk in self._GenerateDescription():
      yield chunk

    yield self.archive_generator.Close()


Instruction:
Create a Python 3 function for
yielding binary chunks respecting archive file headers and footers.

Response:
def _WriteFileChunk(self, chunk):
    """Yields binary chunks, respecting archive file headers and footers.

    Args:
      chunk: the StreamedFileChunk to be written
    """
    if chunk.chunk_index == 0:
      # Make sure size of the original file is passed. It's required
      # when output_writer is StreamingTarWriter.
      st = os.stat_result((0o644, 0, 0, 0, 0, 0, chunk.total_size, 0, 0, 0))
      target_path = _ClientPathToString(chunk.client_path, prefix=self.prefix)
      yield self.archive_generator.WriteFileHeader(target_path, st=st)

    yield self.archive_generator.WriteFileChunk(chunk.data)

    if chunk.chunk_index == chunk.total_chunks - 1:
      yield self.archive_generator.WriteFileFooter()
      self.archived_files.add(chunk.client_path)


Instruction:
Given the following Python 3 function, write the documentation
def DefineFromProtobuf(cls, protobuf):
  mro_chain = [c.__name__ for c in inspect.getmro(cls)]
  if (protobuf.__name__ not in mro_chain and
      not getattr(cls, "allow_custom_class_name", False)):
    raise ProtobufNameMustMatchClassOrParentClassError(
        "Can't define RDFProtoStruct class %s from proto %s "
        "(proto name must match one of the classes in the MRO chain: %s)" %
        (cls.__name__, protobuf.__name__, ", ".join(mro_chain)))

  cls.recorded_rdf_deps = set()

  # Parse message level options.
  message_options = protobuf.DESCRIPTOR.GetOptions()
  semantic_options = message_options.Extensions[semantic_pb2.semantic]

  # Hack to avoid dependency loop.
  # TODO(amoser): remove this hack
  classes_dict = type_info.TypeInfoObject.classes

  # Support message descriptions
  if semantic_options.description and not cls.__doc__:
    cls.__doc__ = semantic_options.description

  cls.union_field = semantic_options.union_field or None

  # We search through all the field descriptors and build type info
  # descriptors from them.
  for field in protobuf.DESCRIPTOR.fields:
    type_descriptor = None

    # Does this field have semantic options?
    options = field.GetOptions().Extensions[semantic_pb2.sem_type]
    kwargs = dict(
        description=options.description,
        name=field.name,
        friendly_name=options.friendly_name,
        field_number=field.number,
        labels=list(options.label))

    if field.has_default_value:
      kwargs["default"] = field.default_value

    # This field is a non-protobuf semantic value.
    if options.type and field.type != TYPE_MESSAGE:
      cls.recorded_rdf_deps.add(options.type)
      rdf_type = rdfvalue.RDFValue.classes.get(options.type)
      if rdf_type:
        if (CHECK_PROTOBUF_DEPENDENCIES and rdf_type not in cls.rdf_deps and
            options.type not in cls.rdf_deps):
          raise rdfvalue.InitializeError(
              "%s.%s: field %s is of type %s, "
              "but type is missing from its dependencies list" %
              (cls.__module__, cls.__name__, field.name, options.type))

        # Make sure that the field type is the same as what is required by the
        # semantic type.
        required_field_type = _SEMANTIC_PRIMITIVE_TO_FIELD_TYPE[
            rdf_type.data_store_type]

        if required_field_type != field.type:
          raise rdfvalue.InitializeError(
              ("%s: .proto file uses incorrect field to store Semantic Value "
               "%s: Should be %s") %
              (cls.__name__, field.name, rdf_type.data_store_type))

      type_descriptor = classes_dict["ProtoRDFValue"](
          rdf_type=options.type, **kwargs)

    # A semantic protobuf is already a semantic value so it is an error to
    # specify it in two places.
    elif options.type and field.type == TYPE_MESSAGE:
      raise rdfvalue.InitializeError(
          ("%s: .proto file specified both Semantic Value type %s and "
           "Semantic protobuf %s") %
          (cls.__name__, options.type, field.message_type.name))

    # Try to figure out what this field actually is from the descriptor.
    elif field.type == TYPE_DOUBLE:
      type_descriptor = classes_dict["ProtoDouble"](**kwargs)

    elif field.type == TYPE_FLOAT:
      type_descriptor = classes_dict["ProtoFloat"](**kwargs)

    elif field.type == TYPE_BOOL:
      type_descriptor = classes_dict["ProtoBoolean"](**kwargs)

    elif field.type == TYPE_STRING:
      type_descriptor = classes_dict["ProtoString"](**kwargs)

    elif field.type == TYPE_BYTES:
      type_descriptor = classes_dict["ProtoBinary"](**kwargs)
      if options.dynamic_type:
        # This may be a dynamic type. In this case the dynamic_type option
        # names a method (which must exist) which should return the class of
        # the embedded semantic value.
        dynamic_cb = getattr(cls, options.dynamic_type, None)
        if dynamic_cb is not None:
          type_descriptor = classes_dict["ProtoDynamicEmbedded"](
              dynamic_cb=dynamic_cb, **kwargs)
        else:
          logging.warning("Dynamic type specifies a non existant callback %s",
                          options.dynamic_type)

    elif (field.type == TYPE_MESSAGE and field.message_type.name == "Any"):
      dynamic_cb = getattr(cls, options.dynamic_type, None)
      type_descriptor = classes_dict["ProtoDynamicAnyValueEmbedded"](
          dynamic_cb=dynamic_cb, **kwargs)

    elif field.type == TYPE_INT64 or field.type == TYPE_INT32:
      type_descriptor = classes_dict["ProtoSignedInteger"](**kwargs)

    elif field.type == TYPE_UINT32 or field.type == TYPE_UINT64:
      type_descriptor = classes_dict["ProtoUnsignedInteger"](**kwargs)

    # An embedded protocol buffer.
    elif field.type == TYPE_MESSAGE and field.message_type:
      # Refer to another protobuf. Note that the target does not need to be
      # known at this time. It will be resolved using the late binding algorithm
      # when it is known. Therefore this can actually also refer to this current
      # protobuf (i.e. nested proto).
      type_descriptor = classes_dict["ProtoEmbedded"](
          nested=field.message_type.name, **kwargs)

      cls.recorded_rdf_deps.add(field.message_type.name)
      if CHECK_PROTOBUF_DEPENDENCIES:
        found = False
        for d in cls.rdf_deps:
          if (hasattr(d, "__name__") and d.__name__ == field.message_type.name
              or d == field.message_type.name):
            found = True

        if not found:
          raise rdfvalue.InitializeError(
              "%s.%s: TYPE_MESSAGE field %s is %s, "
              "but type is missing from its dependencies list" %
              (cls.__module__, cls.__name__, field.name,
               field.message_type.name))

      # TODO(user): support late binding here.
      if type_descriptor.type:
        # This traps the following problem:
        # class Certificate(rdf_protodict.RDFValueArray):
        #    protobuf = jobs_pb2.BlobArray
        #

        # A primitive Protobuf definition like:
        # message Certificate {
        #   ....
        # };

        # And a field like:
        # optional Certificate csr = 1 [(sem_type) = {
        #   description: "A Certificate RDFValue with the CSR in it.",
        # }];

        # If we blindly allowed the Certificate RDFValue to be used, the
        # semantic library will end up embedding a BlobArray protobuf, but the
        # primitive library will still use Certificate.

        # The name of the primitive protobuf the semantic type implements.
        semantic_protobuf_primitive = type_descriptor.type.protobuf.__name__

        # This is an error because the primitive library will use the protobuf
        # named in the field, but the semantic library will implement a
        # different protobuf.
        if semantic_protobuf_primitive != field.message_type.name:
          raise rdfvalue.InitializeError(
              ("%s.%s: Conflicting primitive (%s) and semantic protobuf %s "
               "which implements primitive protobuf (%s)") %
              (cls.__name__, field.name, field.message_type.name,
               type_descriptor.type.__name__, semantic_protobuf_primitive))

    elif field.enum_type:  # It is an enum.
      # TODO(hanuszczak): Protobuf descriptors use `bytes` objects to represent
      # string values. Hence, we add additional `unicode` calls to convert them.
      # It should be investigated whether this behaviour is needed in Python 3
      # as well.

      enum_desc = field.enum_type
      enum_desc_name = str(enum_desc.name)
      enum_dict = {}
      enum_descriptions = {}
      enum_labels = {}

      for enum_value in enum_desc.values:
        enum_value_name = str(enum_value.name)

        enum_dict[enum_value_name] = enum_value.number
        description = enum_value.GetOptions().Extensions[
            semantic_pb2.description]
        enum_descriptions[enum_value_name] = description
        labels = [
            label
            for label in enum_value.GetOptions().Extensions[semantic_pb2.label]
        ]
        enum_labels[enum_value_name] = labels

      type_descriptor = classes_dict["ProtoEnum"](
          enum_name=enum_desc_name,
          enum=enum_dict,
          enum_descriptions=enum_descriptions,
          enum_labels=enum_labels,
          **kwargs)

      # Attach the enum container to the class for easy reference:
      setattr(cls, enum_desc_name, type_descriptor.enum_container)

    # If we do not recognize the type descriptor we ignore this field.
    if type_descriptor is not None:
      # If the field is repeated, wrap it in a ProtoList.
      if field.label == LABEL_REPEATED:
        options = field.GetOptions().Extensions[semantic_pb2.sem_type]
        type_descriptor = classes_dict["ProtoList"](
            type_descriptor, labels=list(options.label))

      try:
        cls.AddDescriptor(type_descriptor)
      except Exception:
        logging.error("Failed to parse protobuf %s", cls)
        raise

    else:
      logging.error("Unknown field type for %s - Ignoring.", field.name)

  if hasattr(cls, "rdf_deps"):
    leftover_deps = set()
    for d in cls.rdf_deps:
      try:
        leftover_deps.add(d.__name__)
      except AttributeError:
        leftover_deps.add(d)
    for d in cls.recorded_rdf_deps:
      leftover_deps.remove(d)
    if leftover_deps:
      raise rdfvalue.InitializeError(
          "Found superfluous dependencies for %s: %s" %
          (cls.__name__, ",".join(leftover_deps)))

Response: Define from an existing protobuf file.


Instruction:
Write a Python 3 script to
start all the component in their own threads.

Response:
def main(argv):
  """Sets up all the component in their own threads."""

  if flags.FLAGS.version:
    print("GRR server {}".format(config_server.VERSION["packageversion"]))
    return

  # We use .startswith so that multiple copies of services can easily be
  # created using systemd as worker1 worker2 ... worker25 etc.

  if not flags.FLAGS.component:
    raise ValueError("Need to specify which component to start.")

  # Start as a worker.
  if flags.FLAGS.component.startswith("worker"):
    worker.main([argv])

  # Start as a frontend that clients communicate with.
  elif flags.FLAGS.component.startswith("frontend"):
    frontend.main([argv])

  # Start as an AdminUI.
  elif flags.FLAGS.component.startswith("admin_ui"):
    admin_ui.main([argv])

  # Raise on invalid component.
  else:
    raise ValueError("No valid component specified. Got: "
                     "%s." % flags.FLAGS.component)


Instruction:
Can you tell what is the following Python 3 function doing
def Operation(self, x, y):
    if x in y:
      return True

    # x might be an iterable
    # first we need to skip strings or we'll do silly things
    if isinstance(x, string_types) or isinstance(x, bytes):
      return False

    try:
      for value in x:
        if value not in y:
          return False
      return True
    except TypeError:
      return False

Response: Whether x is fully contained in y.


Instruction:
How would you code a function in Python 3 to
call when at a leaf value. Should yield a value.

Response:
def _AtLeaf(self, attr_value):
    """Called when at a leaf value. Should yield a value."""
    if isinstance(attr_value, collections.Mapping):
      # If the result is a dict, return each key/value pair as a new dict.
      for k, v in iteritems(attr_value):
        yield {k: v}
    else:
      yield attr_value


Instruction:
Given the following Python 3 function, write the documentation
def _AtNonLeaf(self, attr_value, path):
    try:
      if isinstance(attr_value, collections.Mapping):
        # If it's dictionary-like, treat the dict key as the attribute..
        sub_obj = attr_value.get(path[1])
        if len(path) > 2:
          # Expand any additional elements underneath the key.
          sub_obj = self.Expand(sub_obj, path[2:])
        if isinstance(sub_obj, string_types):
          # If it is a string, stop here
          yield sub_obj
        elif isinstance(sub_obj, collections.Mapping):
          # If the result is a dict, return each key/value pair as a new dict.
          for k, v in iteritems(sub_obj):
            yield {k: v}
        else:
          for value in sub_obj:
            yield value
      else:
        # If it's an iterable, we recurse on each value.
        for sub_obj in attr_value:
          for value in self.Expand(sub_obj, path[1:]):
            yield value
    except TypeError:  # This is then not iterable, we recurse with the value
      for value in self.Expand(attr_value, path[1:]):
        yield value

Response: Recursively yield the values at the given path.


Instruction:
Can you tell what is the following Python 3 function doing
def Compile(self, filter_implemention):
    operator = self.operator.lower()
    if operator == "and" or operator == "&&":
      method = "AndFilter"
    elif operator == "or" or operator == "||":
      method = "OrFilter"
    else:
      raise ParseError("Invalid binary operator %s" % operator)

    args = [x.Compile(filter_implemention) for x in self.args]
    return filter_implemention.FILTERS[method](arguments=args)

Response: Compile the binary expression into a filter object.


Instruction:
How would you code a function in Python 3 to
insert an arg to the current expression.

Response:
def InsertArg(self, string="", **_):
    """Insert an arg to the current expression."""
    if self.state == "LIST_ARG":
      self.list_args.append(string)
    elif self.current_expression.AddArg(string):
      # This expression is complete
      self.stack.append(self.current_expression)
      self.current_expression = self.expression_cls()
      # We go to the BINARY state, to find if there's an AND or OR operator
      return "BINARY"


Instruction:
How would you explain what the following Python 3 function does
def InsertFloatArg(self, string="", **_):
    try:
      float_value = float(string)
      return self.InsertArg(float_value)
    except (TypeError, ValueError):
      raise ParseError("%s is not a valid float." % string)

Response: Inserts a Float argument.


Instruction:
Can you implement a function in Python 3 that
inserts an Integer in base16 argument.

Response:
def InsertInt16Arg(self, string="", **_):
    """Inserts an Integer in base16 argument."""
    try:
      int_value = int(string, 16)
      return self.InsertArg(int_value)
    except (TypeError, ValueError):
      raise ParseError("%s is not a valid base16 integer." % string)


Instruction:
Can you create a Python 3 function that
escapes backslashes found inside a string quote.

Response:
def StringEscape(self, string, match, **_):
    """Escape backslashes found inside a string quote.

    Backslashes followed by anything other than [\'"rnbt] will raise an Error.

    Args:
      string: The string that matched.
      match: The match object (m.group(1) is the escaped code)

    Raises:
      ParseError: For strings other than those used to define a regexp, raise an
        error if the escaped string is not one of [\'"rnbt].
    """
    precondition.AssertType(string, Text)

    # Allow unfiltered strings for regexp operations so that escaped special
    # characters (e.g. \*) or special sequences (e.g. \w) can be used in
    # objectfilter.
    if self.current_expression.operator == "regexp":
      self.string += compatibility.UnescapeString(string)
    elif match.group(1) in "\\'\"rnbt":
      self.string += compatibility.UnescapeString(string)
    else:
      raise ParseError("Invalid escape character %s." % string)


Instruction:
Given the following Python 3 function, write the documentation
def HexEscape(self, string, match, **_):
    hex_string = match.group(1)
    try:
      self.string += binascii.unhexlify(hex_string).decode("utf-8")
    # TODO: In Python 2 `binascii` throws `TypeError` for invalid
    # input values (for whathever reason). This behaviour is fixed in Python 3
    # where `binascii.Error` (a subclass of `ValueError`) is raised. Once we do
    # not have to support Python 2 anymore, this `TypeError` catch should be
    # removed.
    except (binascii.Error, TypeError) as error:
      raise ParseError("Invalid hex escape '{}': {}".format(hex_string, error))

Response: Converts a hex escaped string.


Instruction:
How would you implement a function in Python 3 that
generates a CSRF token based on a secret key id and time.

Response:
def GenerateCSRFToken(user_id, time):
  """Generates a CSRF token based on a secret key, id and time."""
  precondition.AssertType(user_id, Text)
  precondition.AssertOptionalType(time, int)

  time = time or rdfvalue.RDFDatetime.Now().AsMicrosecondsSinceEpoch()

  secret = config.CONFIG.Get("AdminUI.csrf_secret_key", None)
  # TODO(amoser): Django is deprecated. Remove this at some point.
  if not secret:
    secret = config.CONFIG["AdminUI.django_secret_key"]
  digester = hmac.new(secret.encode("ascii"), digestmod=hashlib.sha256)
  digester.update(user_id.encode("ascii"))
  digester.update(CSRF_DELIMITER)
  digester.update(str(time).encode("ascii"))
  digest = digester.digest()

  token = base64.urlsafe_b64encode(b"%s%s%d" % (digest, CSRF_DELIMITER, time))
  return token.rstrip(b"=")


Instruction:
How would you explain what the following Python 3 function does
def StoreCSRFCookie(user, response):

  csrf_token = GenerateCSRFToken(user, None)
  response.set_cookie(
      "csrftoken", csrf_token, max_age=CSRF_TOKEN_DURATION.seconds)

Response: Decorator for WSGI handler that inserts CSRF cookie into response.


Instruction:
Make a summary of the following Python 3 code
def ValidateCSRFTokenOrRaise(request):

  # CSRF check doesn't make sense for GET/HEAD methods, because they can
  # (and are) used when downloading files through <a href> links - and
  # there's no way to set X-CSRFToken header in this case.
  if request.method in ("GET", "HEAD"):
    return

  # In the ideal world only JavaScript can be used to add a custom header, and
  # only within its origin. By default, browsers don't allow JavaScript to
  # make cross origin requests.
  #
  # Unfortunately, in the real world due to bugs in browsers plugins, it can't
  # be guaranteed that a page won't set an HTTP request with a custom header
  # set. That's why we also check the contents of a header via an HMAC check
  # with a server-stored secret.
  #
  # See for more details:
  # https://www.owasp.org/index.php/Cross-Site_Request_Forgery_(CSRF)_Prevention_Cheat_Sheet
  # (Protecting REST Services: Use of Custom Request Headers).
  csrf_token = request.headers.get("X-CSRFToken", "").encode("ascii")
  if not csrf_token:
    logging.info("Did not find headers CSRF token for: %s", request.path)
    raise werkzeug_exceptions.Forbidden("CSRF token is missing")

  try:
    decoded = base64.urlsafe_b64decode(csrf_token + b"==")
    digest, token_time = decoded.rsplit(CSRF_DELIMITER, 1)
    token_time = int(token_time)
  except (TypeError, ValueError):
    logging.info("Malformed CSRF token for: %s", request.path)
    raise werkzeug_exceptions.Forbidden("Malformed CSRF token")

  if len(digest) != hashlib.sha256().digest_size:
    logging.info("Invalid digest size for: %s", request.path)
    raise werkzeug_exceptions.Forbidden("Malformed CSRF token digest")

  expected = GenerateCSRFToken(request.user, token_time)
  if not constant_time.bytes_eq(csrf_token, expected):
    logging.info("Non-matching CSRF token for: %s", request.path)
    raise werkzeug_exceptions.Forbidden("Non-matching CSRF token")

  current_time = rdfvalue.RDFDatetime.Now().AsMicrosecondsSinceEpoch()
  if current_time - token_time > CSRF_TOKEN_DURATION.microseconds:
    logging.info("Expired CSRF token for: %s", request.path)
    raise werkzeug_exceptions.Forbidden("Expired CSRF token")

Response: Decorator for WSGI handler that checks CSRF token against the request.


Instruction:
Can you tell what is the following Python 3 function doing
def _BuildToken(self, request, execution_time):
    token = access_control.ACLToken(
        username=request.user,
        reason=request.args.get("reason", ""),
        process="GRRAdminUI",
        expiry=rdfvalue.RDFDatetime.Now() + execution_time)

    for field in ["Remote_Addr", "X-Forwarded-For"]:
      remote_addr = request.headers.get(field, "")
      if remote_addr:
        token.source_ips.append(remote_addr)
    return token

Response: Builds an ACLToken from the request.


Instruction:
Make a summary of the following Python 3 code
def _HandleHomepage(self, request):

    _ = request

    env = jinja2.Environment(
        loader=jinja2.FileSystemLoader(config.CONFIG["AdminUI.template_root"]),
        autoescape=True)

    create_time = psutil.Process(os.getpid()).create_time()
    context = {
        "heading":
            config.CONFIG["AdminUI.heading"],
        "report_url":
            config.CONFIG["AdminUI.report_url"],
        "help_url":
            config.CONFIG["AdminUI.help_url"],
        "timestamp":
            utils.SmartStr(create_time),
        "use_precompiled_js":
            config.CONFIG["AdminUI.use_precompiled_js"],
        # Used in conjunction with FirebaseWebAuthManager.
        "firebase_api_key":
            config.CONFIG["AdminUI.firebase_api_key"],
        "firebase_auth_domain":
            config.CONFIG["AdminUI.firebase_auth_domain"],
        "firebase_auth_provider":
            config.CONFIG["AdminUI.firebase_auth_provider"],
        "grr_version":
            config.CONFIG["Source.version_string"]
    }
    template = env.get_template("base.html")
    response = werkzeug_wrappers.Response(
        template.render(context), mimetype="text/html")

    # For a redirect-based Firebase authentication scheme we won't have any
    # user information at this point - therefore checking if the user is
    # present.
    try:
      StoreCSRFCookie(request.user, response)
    except RequestHasNoUser:
      pass

    return response

Response: Renders GRR home page by rendering base. html Jinja template.


Instruction:
Write a Python 3 function that can
redirect to GitHub - hosted documentation.

Response:
def _RedirectToRemoteHelp(self, path):
    """Redirect to GitHub-hosted documentation."""
    allowed_chars = set(string.ascii_letters + string.digits + "._-/")
    if not set(path) <= allowed_chars:
      raise RuntimeError("Unusual chars in path %r - "
                         "possible exploit attempt." % path)

    target_path = os.path.join(config.CONFIG["AdminUI.docs_location"], path)

    # We have to redirect via JavaScript to have access to and to preserve the
    # URL hash. We don't know the hash part of the url on the server.
    return werkzeug_wrappers.Response(
        """
<script>
var friendly_hash = window.location.hash;
window.location = '%s' + friendly_hash;
</script>
""" % target_path,
        mimetype="text/html")


Instruction:
Explain what the following Python 3 code does
def WSGIHandler(self):
    sdm = werkzeug_wsgi.SharedDataMiddleware(self, {
        "/": config.CONFIG["AdminUI.document_root"],
    })
    # Use DispatcherMiddleware to make sure that SharedDataMiddleware is not
    # used at all if the URL path doesn't start with "/static". This is a
    # workaround for cases when unicode URLs are used on systems with
    # non-unicode filesystems (as detected by Werkzeug). In this case
    # SharedDataMiddleware may fail early while trying to convert the
    # URL into the file path and not dispatch the call further to our own
    # WSGI handler.
    return werkzeug_wsgi.DispatcherMiddleware(self, {
        "/static": sdm,
    })

Response: Returns GRR s WSGI handler.


Instruction:
Can you create a Python 3 function that
imports the plugins once only.

Response:
def RunOnce(self):
    """Import the plugins once only."""
    # pylint: disable=unused-variable,g-import-not-at-top
    from grr_response_server.gui import gui_plugins
    # pylint: enable=unused-variable,g-import-not-at-top

    if config.CONFIG.Get("AdminUI.django_secret_key", None):
      logging.warning(
          "The AdminUI.django_secret_key option has been deprecated, "
          "please use AdminUI.csrf_secret_key instead.")


Instruction:
Can you implement a function in Python 3 that
lists all registered Grr binaries.

Response:
def ListGrrBinaries(context=None):
  """Lists all registered Grr binaries."""

  items = context.SendIteratorRequest("ListGrrBinaries", None)
  return utils.MapItemsIterator(
      lambda data: GrrBinary(data=data, context=context), items)


Instruction:
Can you tell what is the following Python 3 function doing
def Execute(self, message):
    self.message = message
    if message:
      self.require_fastpoll = message.require_fastpoll

    args = None
    try:
      if self.message.args_rdf_name:
        if not self.in_rdfvalue:
          raise RuntimeError("Did not expect arguments, got %s." %
                             self.message.args_rdf_name)

        if self.in_rdfvalue.__name__ != self.message.args_rdf_name:
          raise RuntimeError(
              "Unexpected arg type %s != %s." %
              (self.message.args_rdf_name, self.in_rdfvalue.__name__))

        args = self.message.payload

      # Only allow authenticated messages in the client
      if self._authentication_required and (
          self.message.auth_state !=
          rdf_flows.GrrMessage.AuthorizationState.AUTHENTICATED):
        raise RuntimeError("Message for %s was not Authenticated." %
                           self.message.name)

      self.cpu_start = self.proc.cpu_times()
      self.cpu_limit = self.message.cpu_limit

      if getattr(flags.FLAGS, "debug_client_actions", False):
        pdb.set_trace()

      try:
        self.Run(args)

      # Ensure we always add CPU usage even if an exception occurred.
      finally:
        used = self.proc.cpu_times()
        self.cpu_used = (used.user - self.cpu_start.user,
                         used.system - self.cpu_start.system)

    except NetworkBytesExceededError as e:
      self.SetStatus(rdf_flows.GrrStatus.ReturnedStatus.NETWORK_LIMIT_EXCEEDED,
                     "%r: %s" % (e, e), traceback.format_exc())

    # We want to report back all errors and map Python exceptions to
    # Grr Errors.
    except Exception as e:  # pylint: disable=broad-except
      self.SetStatus(rdf_flows.GrrStatus.ReturnedStatus.GENERIC_ERROR,
                     "%r: %s" % (e, e), traceback.format_exc())

      if flags.FLAGS.pdb_post_mortem:
        self.DisableNanny()
        pdb.post_mortem()

    if self.status.status != rdf_flows.GrrStatus.ReturnedStatus.OK:
      logging.info("Job Error (%s): %s", self.__class__.__name__,
                   self.status.error_message)

      if self.status.backtrace:
        logging.debug(self.status.backtrace)

    if self.cpu_used:
      self.status.cpu_time_used.user_cpu_time = self.cpu_used[0]
      self.status.cpu_time_used.system_cpu_time = self.cpu_used[1]

    # This returns the error status of the Actions to the flow.
    self.SendReply(self.status, message_type=rdf_flows.GrrMessage.Type.STATUS)

    self._RunGC()

Response: This function parses the RDFValue from the server and runs the appropriate method.


Instruction:
Given the following Python 3 function, write the documentation
def SetStatus(self, status, message="", backtrace=None):
    self.status.status = status
    self.status.error_message = utils.SmartUnicode(message)
    if backtrace:
      self.status.backtrace = utils.SmartUnicode(backtrace)

Response: Sets a status to report back to the server.


Instruction:
Create a Python 3 function to
send a reply to the server.

Response:
def SendReply(self,
                rdf_value=None,
                session_id=None,
                message_type=rdf_flows.GrrMessage.Type.MESSAGE):
    """Send response back to the server."""
    # TODO(hanuszczak): This is pretty bad. Here we assume that if the session
    # id is not none we are "replying" to a well-known flow. If we are replying
    # to a well-known flow we cannot increment the response id (since these are
    # not really responses) because otherwise the parent flow will think it is
    # missing some responses.
    #
    # Moreover, the message queue (at least in the flow test helper workflow)
    # expects flow messages to have request id equal to 0. This is rather
    # convoluted and should be untangled in the future.
    #
    # Once we have a separate method for communicating with well-known flows
    # this if-statement should no longer be relevant (since setting custom
    # session ids would become illegal).
    if session_id is None:
      response_id = self.response_id
      request_id = self.message.request_id
      session_id = self.message.session_id
      self.response_id += 1
    else:
      response_id = 0
      request_id = 0

    self.grr_worker.SendReply(
        rdf_value,
        # This is not strictly necessary but adds context
        # to this response.
        name=self.__class__.__name__,
        session_id=session_id,
        response_id=response_id,
        request_id=request_id,
        message_type=message_type,
        task_id=self.message.Get("task_id") or None,
        require_fastpoll=self.require_fastpoll)


Instruction:
Explain what the following Python 3 code does
def Progress(self):
    now = time.time()
    if now - self.last_progress_time <= 2:
      return

    self.last_progress_time = now

    # Prevent the machine from sleeping while the action is running.
    client_utils.KeepAlive()

    self.grr_worker.Heartbeat()

    user_start = self.cpu_start.user
    system_start = self.cpu_start.system
    cpu_times = self.proc.cpu_times()
    user_end = cpu_times.user
    system_end = cpu_times.system

    used_cpu = user_end - user_start + system_end - system_start

    if used_cpu > self.cpu_limit:
      self.grr_worker.SendClientAlert("Cpu limit exceeded.")
      raise CPUExceededError("Action exceeded cpu limit.")

Response: Indicate progress of the client action.


Instruction:
Implement a Python 3 function for
writing the provided graphs to the DB with the given client label and timestamp.

Response:
def WriteClientGraphSeries(
      self,
      graph_series,
      client_label,
      timestamp,
      cursor=None,
  ):
    """Writes the provided graphs to the DB with the given client label."""
    args = {
        "client_label": client_label,
        "report_type": graph_series.report_type.SerializeToDataStore(),
        "timestamp": mysql_utils.RDFDatetimeToTimestamp(timestamp),
        "graph_series": graph_series.SerializeToString(),
    }

    query = """
      INSERT INTO client_report_graphs (client_label, report_type, timestamp,
                                        graph_series)
      VALUES (%(client_label)s, %(report_type)s, FROM_UNIXTIME(%(timestamp)s),
              %(graph_series)s)
      ON DUPLICATE KEY UPDATE graph_series = VALUES(graph_series)
    """

    cursor.execute(query, args)


Instruction:
Here you have a function in Python 3, explain what it does
def ReadAllClientGraphSeries(
      self,
      client_label,
      report_type,
      time_range = None,
      cursor=None):
    """Reads graph series for the given label and report-type from the DB."""
    query = """
      SELECT UNIX_TIMESTAMP(timestamp), graph_series
      FROM client_report_graphs
      WHERE client_label = %s AND report_type = %s
    """
    args = [client_label, report_type.SerializeToDataStore()]

    if time_range is not None:
      query += "AND `timestamp` BETWEEN FROM_UNIXTIME(%s) AND FROM_UNIXTIME(%s)"
      args += [
          mysql_utils.RDFDatetimeToTimestamp(time_range.start),
          mysql_utils.RDFDatetimeToTimestamp(time_range.end)
      ]

    cursor.execute(query, args)
    results = {}
    for timestamp, raw_series in cursor.fetchall():
      # TODO(hanuszczak): pytype does not seem to understand overloads, so it is
      # not possible to correctly annotate `TimestampToRDFDatetime`.
      timestamp = cast(rdfvalue.RDFDatetime,
                       mysql_utils.TimestampToRDFDatetime(timestamp))

      series = rdf_stats.ClientGraphSeries.FromSerializedString(raw_series)
      results[timestamp] = series
    return results

Response: Reads graph series for the given label and report - type from the DB.


Instruction:
Write a Python 3 function for
reading the latest graph series for a given client - label.

Response:
def ReadMostRecentClientGraphSeries(
      self,
      client_label,
      report_type,
      cursor=None):
    """Fetches the latest graph series for a client-label from the DB."""
    query = """
      SELECT graph_series
      FROM client_report_graphs
      WHERE client_label = %s AND report_type = %s
      ORDER BY timestamp DESC
      LIMIT 1
    """
    args = [client_label, report_type.SerializeToDataStore()]
    cursor.execute(query, args)
    result = cursor.fetchone()
    if result is None:
      return None
    else:
      return rdf_stats.ClientGraphSeries.FromSerializedString(result[0])


Instruction:
Implement a function in Python 3 to
convert given RDFValue to an RDFURN of a file to be downloaded.

Response:
def CollectionItemToAff4Path(item, client_id=None):
  """Converts given RDFValue to an RDFURN of a file to be downloaded."""
  if isinstance(item, rdf_flows.GrrMessage):
    client_id = item.source
    item = item.payload
  elif isinstance(item, rdf_flow_objects.FlowResult):
    client_id = item.client_id
    item = item.payload

  if not client_id:
    raise ValueError("Could not determine client_id.")
  elif isinstance(client_id, string_types):
    client_id = rdf_client.ClientURN(client_id)

  if isinstance(item, rdf_client_fs.StatEntry):
    return item.AFF4Path(client_id)
  elif isinstance(item, rdf_file_finder.FileFinderResult):
    return item.stat_entry.AFF4Path(client_id)
  elif isinstance(item, collectors.ArtifactFilesDownloaderResult):
    if item.HasField("downloaded_file"):
      return item.downloaded_file.AFF4Path(client_id)

  raise ItemNotExportableError(item)


Instruction:
Can you implement a function in Python 3 that
converts given RDFValue to a ClientPath of a file to be downloaded.

Response:
def CollectionItemToClientPath(item, client_id=None):
  """Converts given RDFValue to a ClientPath of a file to be downloaded."""
  if isinstance(item, rdf_flows.GrrMessage):
    client_id = item.source
    item = item.payload
  elif isinstance(item, rdf_flow_objects.FlowResult):
    client_id = item.client_id
    item = item.payload

  if client_id is None:
    raise ValueError("Could not determine client_id.")
  elif isinstance(client_id, rdfvalue.RDFURN):
    client_id = client_id.Basename()

  if isinstance(item, rdf_client_fs.StatEntry):
    return db.ClientPath.FromPathSpec(client_id, item.pathspec)
  elif isinstance(item, rdf_file_finder.FileFinderResult):
    return db.ClientPath.FromPathSpec(client_id, item.stat_entry.pathspec)
  elif isinstance(item, collectors.ArtifactFilesDownloaderResult):
    if item.HasField("downloaded_file"):
      return db.ClientPath.FromPathSpec(client_id,
                                        item.downloaded_file.pathspec)

  raise ItemNotExportableError(item)


Instruction:
Implement a function in Python 3 to
return a knowledgebase from an rdf client object.

Response:
def GetKnowledgeBase(rdf_client_obj, allow_uninitialized=False):
  """Returns a knowledgebase from an rdf client object."""
  if not allow_uninitialized:
    if rdf_client_obj is None:
      raise artifact_utils.KnowledgeBaseUninitializedError(
          "No client snapshot given.")
    if rdf_client_obj.knowledge_base is None:
      raise artifact_utils.KnowledgeBaseUninitializedError(
          "KnowledgeBase empty for %s." % rdf_client_obj.client_id)
    kb = rdf_client_obj.knowledge_base
    if not kb.os:
      raise artifact_utils.KnowledgeBaseAttributesMissingError(
          "KnowledgeBase missing OS for %s. Knowledgebase content: %s" %
          (rdf_client_obj.client_id, kb))
  if rdf_client_obj is None or rdf_client_obj.knowledge_base is None:
    return rdf_client.KnowledgeBase()

  version = rdf_client_obj.os_version.split(".")
  kb = rdf_client_obj.knowledge_base
  try:
    kb.os_major_version = int(version[0])
    if len(version) >= 1:
      kb.os_minor_version = int(version[1])
  except ValueError:
    pass

  return kb


Instruction:
Write a Python 3 script to
set core values from GRR into the knowledgebase.

Response:
def SetCoreGRRKnowledgeBaseValues(kb, client_obj):
  """Set core values from GRR into the knowledgebase."""
  client_schema = client_obj.Schema
  kb.fqdn = utils.SmartUnicode(client_obj.Get(client_schema.FQDN, ""))
  if not kb.fqdn:
    kb.fqdn = utils.SmartUnicode(client_obj.Get(client_schema.HOSTNAME, ""))
  versions = client_obj.Get(client_schema.OS_VERSION)
  if versions and versions.versions:
    try:
      kb.os_major_version = versions.versions[0]
      kb.os_minor_version = versions.versions[1]
    except IndexError:
      # Some OSs don't have a minor version.
      pass
  client_os = client_obj.Get(client_schema.SYSTEM)
  if client_os:
    kb.os = utils.SmartUnicode(client_obj.Get(client_schema.SYSTEM))


Instruction:
Can you tell what is the following Python 3 function doing
def ApplyParsersToResponses(parser_factory, responses, flow_obj):
  # We have some processors to run.
  knowledge_base = flow_obj.state.knowledge_base

  parsed_responses = []

  if parser_factory.HasSingleResponseParsers():
    for response in responses:
      for parser in parser_factory.SingleResponseParsers():
        parsed_responses.extend(
            parser.ParseResponse(knowledge_base, response,
                                 flow_obj.args.path_type))

  for parser in parser_factory.MultiResponseParsers():
    parsed_responses.extend(parser.ParseResponses(knowledge_base, responses))

  has_single_file_parsers = parser_factory.HasSingleFileParsers()
  has_multi_file_parsers = parser_factory.HasMultiFileParsers()

  if has_single_file_parsers or has_multi_file_parsers:
    precondition.AssertIterableType(responses, rdf_client_fs.StatEntry)
    pathspecs = [response.pathspec for response in responses]
    if data_store.RelationalDBEnabled():
      # TODO(amoser): This is not super efficient, AFF4 provided an api to open
      # all pathspecs at the same time, investigate if optimizing this is worth
      # it.
      filedescs = []
      for pathspec in pathspecs:
        client_path = db.ClientPath.FromPathSpec(flow_obj.client_id, pathspec)
        filedescs.append(file_store.OpenFile(client_path))
    else:
      filedescs = MultiOpenAff4File(flow_obj, pathspecs)

  if has_single_file_parsers:
    for response, filedesc in zip(responses, filedescs):
      for parser in parser_factory.SingleFileParsers():
        parsed_responses.extend(
            parser.ParseFile(knowledge_base, response.pathspec, filedesc))

  if has_multi_file_parsers:
    for parser in parser_factory.MultiFileParsers():
      parsed_responses.extend(
          parser.ParseFiles(knowledge_base, pathspecs, filedescs))

  return parsed_responses or responses

Response: Parse the given list of responses with applicable parsers.


Instruction:
Can you tell what is the following Python 3 function doing
def UploadArtifactYamlFile(file_content,
                           overwrite=True,
                           overwrite_system_artifacts=False):
  """Upload a yaml or json file as an artifact to the datastore."""
  loaded_artifacts = []
  registry_obj = artifact_registry.REGISTRY
  # Make sure all artifacts are loaded so we don't accidentally overwrite one.
  registry_obj.GetArtifacts(reload_datastore_artifacts=True)

  new_artifacts = registry_obj.ArtifactsFromYaml(file_content)
  new_artifact_names = set()
  # A quick syntax check before we upload anything.
  for artifact_value in new_artifacts:
    artifact_registry.ValidateSyntax(artifact_value)
    new_artifact_names.add(artifact_value.name)

  # Iterate through each artifact adding it to the collection.
  artifact_coll = artifact_registry.ArtifactCollection(ARTIFACT_STORE_ROOT_URN)
  current_artifacts = list(artifact_coll)

  # We need to remove artifacts we are overwriting.
  filtered_artifacts = [
      art for art in current_artifacts if art.name not in new_artifact_names
  ]

  artifact_coll.Delete()
  with data_store.DB.GetMutationPool() as pool:
    for artifact_value in filtered_artifacts:
      artifact_coll.Add(artifact_value, mutation_pool=pool)

    for artifact_value in new_artifacts:
      registry_obj.RegisterArtifact(
          artifact_value,
          source="datastore:%s" % ARTIFACT_STORE_ROOT_URN,
          overwrite_if_exists=overwrite,
          overwrite_system_artifacts=overwrite_system_artifacts)

      artifact_coll.Add(artifact_value, mutation_pool=pool)
      if data_store.RelationalDBEnabled():
        data_store.REL_DB.WriteArtifact(artifact_value)

      loaded_artifacts.append(artifact_value)

      name = artifact_value.name
      logging.info("Uploaded artifact %s to %s", name, ARTIFACT_STORE_ROOT_URN)

  # Once all artifacts are loaded we can validate dependencies. Note that we do
  # not have to perform a syntax validation because it is already done after
  # YAML is parsed.
  for artifact_value in loaded_artifacts:
    artifact_registry.ValidateDependencies(artifact_value)

Response: Uploads a yaml file as an artifact to the datastore.


Instruction:
Given the following Python 3 function, write the documentation
def CallLoggedAndAccounted(f):

  @functools.wraps(f)
  def Decorator(*args, **kwargs):
    try:
      start_time = time.time()
      result = f(*args, **kwargs)
      latency = time.time() - start_time

      stats_collector_instance.Get().RecordEvent(
          "db_request_latency", latency, fields=[f.__name__])
      logging.debug("DB request %s SUCCESS (%.3fs)", f.__name__, latency)

      return result
    except db.Error as e:
      stats_collector_instance.Get().IncrementCounter(
          "db_request_errors", fields=[f.__name__, "grr"])
      logging.debug("DB request %s GRR ERROR: %s", f.__name__,
                    utils.SmartUnicode(e))
      raise
    except Exception as e:
      stats_collector_instance.Get().IncrementCounter(
          "db_request_errors", fields=[f.__name__, "db"])
      logging.debug("DB request %s INTERNAL DB ERROR : %s", f.__name__,
                    utils.SmartUnicode(e))
      raise

  return Decorator

Response: Decorator to log and account for a DB call.


Instruction:
Implement a Python 3 function for
escaping wildcard characters for strings intended to be used with LIKE.

Response:
def EscapeWildcards(string):
  """Escapes wildcard characters for strings intended to be used with `LIKE`.

  Databases don't automatically escape wildcard characters ('%', '_'), so any
  non-literal string that is passed to `LIKE` and is expected to match literally
  has to be manually escaped.

  Args:
    string: A string to escape.

  Returns:
    An escaped string.
  """
  precondition.AssertType(string, Text)
  return string.replace("%", r"\%").replace("_", r"\_")


Instruction:
Create a Python 3 function for
converting hunt id string to an integer.

Response:
def HuntIDToInt(hunt_id):
  """Convert hunt id string to an integer."""
  # TODO(user): This code is only needed for a brief period of time when we
  # allow running new rel-db flows with old aff4-based hunts. In this scenario
  # parent_hunt_id is effectively not used, but it has to be an
  # integer. Stripping "H:" from hunt ids then makes the rel-db happy. Remove
  # this code when hunts are rel-db only.
  if hunt_id.startswith("H:"):
    hunt_id = hunt_id[2:]

  try:
    return int(hunt_id or "0", 16)
  except ValueError as e:
    raise HuntIDIsNotAnIntegerError(e)


Instruction:
Can you implement a function in Python 3 that
writes metadata about the client.

Response:
def WriteClientMetadata(self,
                          client_id,
                          certificate=None,
                          fleetspeak_enabled=None,
                          first_seen=None,
                          last_ping=None,
                          last_clock=None,
                          last_ip=None,
                          last_foreman=None):
    """Write metadata about the client."""
    md = {}
    if certificate is not None:
      md["certificate"] = certificate

    if fleetspeak_enabled is not None:
      md["fleetspeak_enabled"] = fleetspeak_enabled

    if first_seen is not None:
      md["first_seen"] = first_seen

    if last_ping is not None:
      md["ping"] = last_ping

    if last_clock is not None:
      md["clock"] = last_clock

    if last_ip is not None:
      md["ip"] = last_ip

    if last_foreman is not None:
      md["last_foreman_time"] = last_foreman

    if not md:
      raise ValueError("NOOP write.")

    self.metadatas.setdefault(client_id, {}).update(md)


Instruction:
Here you have a function in Python 3, explain what it does
def MultiReadClientMetadata(self, client_ids):
    res = {}
    for client_id in client_ids:
      md = self.metadatas.get(client_id, None)
      if md is None:
        continue

      res[client_id] = rdf_objects.ClientMetadata(
          certificate=md.get("certificate"),
          fleetspeak_enabled=md.get("fleetspeak_enabled"),
          first_seen=md.get("first_seen"),
          ping=md.get("ping"),
          clock=md.get("clock"),
          ip=md.get("ip"),
          last_foreman_time=md.get("last_foreman_time"),
          last_crash_timestamp=md.get("last_crash_timestamp"),
          startup_info_timestamp=md.get("startup_info_timestamp"))

    return res

Response: Reads ClientMetadata records for a list of clients.


Instruction:
How would you explain what the following Python 3 function does
def WriteClientSnapshot(self, snapshot):
    client_id = snapshot.client_id

    if client_id not in self.metadatas:
      raise db.UnknownClientError(client_id)

    startup_info = snapshot.startup_info
    snapshot.startup_info = None

    ts = rdfvalue.RDFDatetime.Now()
    history = self.clients.setdefault(client_id, {})
    history[ts] = snapshot.SerializeToString()

    history = self.startup_history.setdefault(client_id, {})
    history[ts] = startup_info.SerializeToString()

    snapshot.startup_info = startup_info

Response: Writes a new client snapshot.


Instruction:
Can you tell what is the following Python 3 function doing
def MultiReadClientSnapshot(self, client_ids):
    res = {}
    for client_id in client_ids:
      history = self.clients.get(client_id, None)
      if not history:
        res[client_id] = None
        continue
      last_timestamp = max(history)
      last_serialized = history[last_timestamp]
      client_obj = rdf_objects.ClientSnapshot.FromSerializedString(
          last_serialized)
      client_obj.timestamp = last_timestamp
      client_obj.startup_info = rdf_client.StartupInfo.FromSerializedString(
          self.startup_history[client_id][last_timestamp])
      res[client_id] = client_obj
    return res

Response: Reads the latest client snapshots for a list of clients.


Instruction:
Implement a function in Python 3 to
read full client information for a list of clients.

Response:
def MultiReadClientFullInfo(self, client_ids, min_last_ping=None):
    """Reads full client information for a list of clients."""
    res = {}
    for client_id in client_ids:
      try:
        md = self.ReadClientMetadata(client_id)
      except db.UnknownClientError:
        continue

      if md and min_last_ping and md.ping < min_last_ping:
        continue
      last_snapshot = self.ReadClientSnapshot(client_id)
      full_info = rdf_objects.ClientFullInfo(
          metadata=md,
          labels=self.ReadClientLabels(client_id),
          last_startup_info=self.ReadClientStartupInfo(client_id))
      if last_snapshot is None:
        full_info.last_snapshot = rdf_objects.ClientSnapshot(
            client_id=client_id)
      else:
        full_info.last_snapshot = last_snapshot
      res[client_id] = full_info
    return res


Instruction:
Write a Python 3 function for
reading the last - ping timestamps for clients in the DB.

Response:
def ReadClientLastPings(self,
                          min_last_ping=None,
                          max_last_ping=None,
                          fleetspeak_enabled=None):
    """Reads last-ping timestamps for clients in the DB."""
    last_pings = {}
    for client_id, metadata in iteritems(self.metadatas):
      last_ping = metadata.get("ping", rdfvalue.RDFDatetime(0))
      is_fleetspeak_client = metadata.get("fleetspeak_enabled", False)
      if min_last_ping is not None and last_ping < min_last_ping:
        continue
      elif max_last_ping is not None and last_ping > max_last_ping:
        continue
      elif (fleetspeak_enabled is not None and
            is_fleetspeak_client != fleetspeak_enabled):
        continue
      else:
        last_pings[client_id] = metadata.get("ping", None)
    return last_pings


Instruction:
Write a Python 3 script for
writing the full history for a particular client.

Response:
def WriteClientSnapshotHistory(self, clients):
    """Writes the full history for a particular client."""
    if clients[0].client_id not in self.metadatas:
      raise db.UnknownClientError(clients[0].client_id)

    for client in clients:
      startup_info = client.startup_info
      client.startup_info = None

      snapshots = self.clients.setdefault(client.client_id, {})
      snapshots[client.timestamp] = client.SerializeToString()

      startup_infos = self.startup_history.setdefault(client.client_id, {})
      startup_infos[client.timestamp] = startup_info.SerializeToString()

      client.startup_info = startup_info


Instruction:
Create a Python 3 function for
reading the full history for a particular client.

Response:
def ReadClientSnapshotHistory(self, client_id, timerange=None):
    """Reads the full history for a particular client."""
    from_time, to_time = self._ParseTimeRange(timerange)

    history = self.clients.get(client_id)
    if not history:
      return []
    res = []
    for ts in sorted(history, reverse=True):
      if ts < from_time or ts > to_time:
        continue

      client_obj = rdf_objects.ClientSnapshot.FromSerializedString(history[ts])
      client_obj.timestamp = ts
      client_obj.startup_info = rdf_client.StartupInfo.FromSerializedString(
          self.startup_history[client_id][ts])
      res.append(client_obj)
    return res


Instruction:
How would you code a function in Python 3 to
associate the provided keywords with the client.

Response:
def AddClientKeywords(self, client_id, keywords):
    """Associates the provided keywords with the client."""
    if client_id not in self.metadatas:
      raise db.UnknownClientError(client_id)

    for kw in keywords:
      self.keywords.setdefault(kw, {})
      self.keywords[kw][client_id] = rdfvalue.RDFDatetime.Now()


Instruction:
Here you have a function in Python 3, explain what it does
def ListClientsForKeywords(self, keywords, start_time=None):
    res = {kw: [] for kw in keywords}
    for kw in keywords:
      for client_id, timestamp in iteritems(self.keywords.get(kw, {})):
        if start_time is not None and timestamp < start_time:
          continue
        res[kw].append(client_id)
    return res

Response: Lists the clients associated with keywords.


Instruction:
Given the following Python 3 function, write the documentation
def RemoveClientKeyword(self, client_id, keyword):
    if keyword in self.keywords and client_id in self.keywords[keyword]:
      del self.keywords[keyword][client_id]

Response: Removes the association of a particular client to a keyword.


Instruction:
How would you implement a function in Python 3 that
adds a user label to a client.

Response:
def AddClientLabels(self, client_id, owner, labels):
    """Attaches a user label to a client."""
    if client_id not in self.metadatas:
      raise db.UnknownClientError(client_id)

    labelset = self.labels.setdefault(client_id, {}).setdefault(owner, set())
    for l in labels:
      labelset.add(utils.SmartUnicode(l))


Instruction:
How would you implement a function in Python 3 that
reads the user labels for a list of clients.

Response:
def MultiReadClientLabels(self, client_ids):
    """Reads the user labels for a list of clients."""
    res = {}
    for client_id in client_ids:
      res[client_id] = []
      owner_dict = self.labels.get(client_id, {})
      for owner, labels in iteritems(owner_dict):
        for l in labels:
          res[client_id].append(rdf_objects.ClientLabel(owner=owner, name=l))
      res[client_id].sort(key=lambda label: (label.owner, label.name))
    return res


Instruction:
Can you generate the documentation for the following Python 3 function
def RemoveClientLabels(self, client_id, owner, labels):
    labelset = self.labels.setdefault(client_id, {}).setdefault(owner, set())
    for l in labels:
      labelset.discard(utils.SmartUnicode(l))

Response: Removes a list of user labels from a given client.


Instruction:
Write a Python 3 script to
read all client labels known to the system.

Response:
def ReadAllClientLabels(self):
    """Lists all client labels known to the system."""
    result = set()
    for labels_dict in itervalues(self.labels):
      for owner, names in iteritems(labels_dict):
        for name in names:
          result.add(rdf_objects.ClientLabel(owner=owner, name=name))

    return list(result)


Instruction:
Write a Python 3 function for
writing a new client startup record.

Response:
def WriteClientStartupInfo(self, client_id, startup_info):
    """Writes a new client startup record."""
    if client_id not in self.metadatas:
      raise db.UnknownClientError(client_id)

    ts = rdfvalue.RDFDatetime.Now()
    self.metadatas[client_id]["startup_info_timestamp"] = ts
    history = self.startup_history.setdefault(client_id, {})
    history[ts] = startup_info.SerializeToString()


Instruction:
How would you explain what the following Python 3 function does
def ReadClientStartupInfo(self, client_id):
    history = self.startup_history.get(client_id, None)
    if not history:
      return None

    ts = max(history)
    res = rdf_client.StartupInfo.FromSerializedString(history[ts])
    res.timestamp = ts
    return res

Response: Reads the latest client startup record for a single client.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def ReadClientStartupInfoHistory(self, client_id, timerange=None):
    from_time, to_time = self._ParseTimeRange(timerange)

    history = self.startup_history.get(client_id)
    if not history:
      return []
    res = []
    for ts in sorted(history, reverse=True):
      if ts < from_time or ts > to_time:
        continue

      client_data = rdf_client.StartupInfo.FromSerializedString(history[ts])
      client_data.timestamp = ts
      res.append(client_data)
    return res

Response: Reads the full startup history for a particular client.


Instruction:
Can you tell what is the following Python 3 function doing
def WriteClientCrashInfo(self, client_id, crash_info):
    if client_id not in self.metadatas:
      raise db.UnknownClientError(client_id)

    ts = rdfvalue.RDFDatetime.Now()
    self.metadatas[client_id]["last_crash_timestamp"] = ts
    history = self.crash_history.setdefault(client_id, {})
    history[ts] = crash_info.SerializeToString()

Response: Writes a new client crash record.


Instruction:
How would you implement a function in Python 3 that
reads the latest client crash record for a single client.

Response:
def ReadClientCrashInfo(self, client_id):
    """Reads the latest client crash record for a single client."""
    history = self.crash_history.get(client_id, None)
    if not history:
      return None

    ts = max(history)
    res = rdf_client.ClientCrash.FromSerializedString(history[ts])
    res.timestamp = ts
    return res


Instruction:
Create a Python 3 function for
reading the full crash history for a particular client.

Response:
def ReadClientCrashInfoHistory(self, client_id):
    """Reads the full crash history for a particular client."""
    history = self.crash_history.get(client_id)
    if not history:
      return []
    res = []
    for ts in sorted(history, reverse=True):
      client_data = rdf_client.ClientCrash.FromSerializedString(history[ts])
      client_data.timestamp = ts
      res.append(client_data)
    return res


Instruction:
How would you explain what the following Python 3 function does
def WriteClientStats(self, client_id,
                       stats):
    """Stores a ClientStats instance."""
    if client_id not in self.ReadAllClientIDs():
      raise db.UnknownClientError(client_id)

    self.client_stats[client_id][rdfvalue.RDFDatetime.Now()] = stats

Response: Stores a ClientStats instance.


Instruction:
Explain what the following Python 3 code does
def ReadClientStats(self, client_id,
                      min_timestamp,
                      max_timestamp
                     ):
    """Reads ClientStats for a given client and time range."""
    results = []
    for timestamp, stats in iteritems(self.client_stats[client_id]):
      if min_timestamp <= timestamp <= max_timestamp:
        results.append(stats)
    return results

Response: Reads ClientStats for a given client and time range.


Instruction:
How would you implement a function in Python 3 that
deletes ClientStats older than a given timestamp.

Response:
def DeleteOldClientStats(self, yield_after_count,
                           retention_time
                          ):
    """Deletes ClientStats older than a given timestamp."""
    deleted_count = 0
    yielded = False

    for stats_dict in itervalues(self.client_stats):
      for timestamp in list(stats_dict.keys()):
        if timestamp < retention_time:
          del stats_dict[timestamp]
          deleted_count += 1

          if deleted_count >= yield_after_count:
            yield deleted_count
            yielded = True
            deleted_count = 0

    if deleted_count > 0 or not yielded:
      yield deleted_count


Instruction:
Write a Python 3 function for
computing client - activity stats for all GRR versions in the DB.

Response:
def CountClientVersionStringsByLabel(self, day_buckets):
    """Computes client-activity stats for all GRR versions in the DB."""

    def ExtractVersion(client_info):
      return client_info.last_snapshot.GetGRRVersionString()

    return self._CountClientStatisticByLabel(day_buckets, ExtractVersion)


Instruction:
Can you tell what is the following Python 3 function doing
def CountClientPlatformsByLabel(self, day_buckets):

    def ExtractPlatform(client_info):
      return client_info.last_snapshot.knowledge_base.os

    return self._CountClientStatisticByLabel(day_buckets, ExtractPlatform)

Response: Computes client - activity stats for all client platforms in the DB.


Instruction:
Write a Python 3 script to
compute client - activity stats for all OS - releases in the DB.

Response:
def CountClientPlatformReleasesByLabel(self, day_buckets):
    """Computes client-activity stats for OS-release strings in the DB."""
    return self._CountClientStatisticByLabel(
        day_buckets, lambda client_info: client_info.last_snapshot.Uname())


Instruction:
Can you tell what is the following Python 3 function doing
def _CountClientStatisticByLabel(self, day_buckets, extract_statistic_fn):
    counts = collections.defaultdict(int)
    now = rdfvalue.RDFDatetime.Now()
    for info in self.IterateAllClientsFullInfo(batch_size=db.MAX_COUNT):
      if not info.metadata.ping:
        continue
      statistic_value = extract_statistic_fn(info)
      for client_label in info.GetLabelsNames(owner="GRR"):
        for day_bucket in day_buckets:
          time_boundary = now - rdfvalue.Duration.FromDays(day_bucket)
          if info.metadata.ping > time_boundary:
            # Count the client if it has been active in the last 'day_bucket'
            # days.
            counts[(statistic_value, client_label, day_bucket)] += 1
    return dict(counts)

Response: Counts the client - activity metrics for a particular statistic.