Instruction:
Here you have a function in Python 3, explain what it does
def ExecuteCommandFromClient(command):
  cmd = command.cmd
  args = command.args
  time_limit = command.time_limit

  res = client_utils_common.Execute(cmd, args, time_limit)
  (stdout, stderr, status, time_used) = res

  # Limit output to 10MB so our response doesn't get too big.
  stdout = stdout[:10 * 1024 * 1024]
  stderr = stderr[:10 * 1024 * 1024]

  yield rdf_client_action.ExecuteResponse(
      request=command,
      stdout=stdout,
      stderr=stderr,
      exit_status=status,
      # We have to return microseconds.
      time_used=int(1e6 * time_used))

Response: Executes one of the predefined commands.


Instruction:
Given the following Python 3 function, write the documentation
def StatFSFromClient(args):
  if platform.system() == "Windows":
    raise RuntimeError("os.statvfs not available on Windows")

  for path in args.path_list:

    try:
      fd = vfs.VFSOpen(rdf_paths.PathSpec(path=path, pathtype=args.pathtype))
      st = fd.StatFS()
      mount_point = fd.GetMountPoint()
    except (IOError, OSError):
      continue

    unix = rdf_client_fs.UnixVolume(mount_point=mount_point)

    # On linux pre 2.6 kernels don't have frsize, so we fall back to bsize.
    # The actual_available_allocation_units attribute is set to blocks
    # available to the unprivileged user, root may have some additional
    # reserved space.
    yield rdf_client_fs.Volume(
        bytes_per_sector=(st.f_frsize or st.f_bsize),
        sectors_per_allocation_unit=1,
        total_allocation_units=st.f_blocks,
        actual_available_allocation_units=st.f_bavail,
        unixvolume=unix)

Response: A generator function that returns a list of rdf_client_fs. UnixVolume objects.


Instruction:
Implement a Python 3 function for
reading a buffer on the client and sends it to the server.

Response:
def Run(self, args):
    """Reads a buffer on the client and sends it to the server."""
    # Make sure we limit the size of our output
    if args.length > constants.CLIENT_MAX_BUFFER_SIZE:
      raise RuntimeError("Can not read buffers this large.")

    try:
      fd = vfs.VFSOpen(args.pathspec, progress_callback=self.Progress)

      fd.Seek(args.offset)
      offset = fd.Tell()

      data = fd.Read(args.length)

    except (IOError, OSError) as e:
      self.SetStatus(rdf_flows.GrrStatus.ReturnedStatus.IOERROR, e)
      return

    # Now return the data to the server
    self.SendReply(
        rdf_client.BufferReference(
            offset=offset, data=data, length=len(data), pathspec=fd.pathspec))


Instruction:
How would you explain what the following Python 3 function does
def Run(self, args):
    # Make sure we limit the size of our output
    if args.length > constants.CLIENT_MAX_BUFFER_SIZE:
      raise RuntimeError("Can not read buffers this large.")

    data = vfs.ReadVFS(
        args.pathspec,
        args.offset,
        args.length,
        progress_callback=self.Progress)
    result = rdf_protodict.DataBlob(
        data=zlib.compress(data),
        compression=rdf_protodict.DataBlob.CompressionType.ZCOMPRESSION)

    digest = hashlib.sha256(data).digest()

    # Ensure that the buffer is counted against this response. Check network
    # send limit.
    self.ChargeBytesToSession(len(data))

    # Now return the data to the server into the special TransferStore well
    # known flow.
    self.grr_worker.SendReply(
        result, session_id=rdfvalue.SessionID(flow_name="TransferStore"))

    # Now report the hash of this blob to our flow as well as the offset and
    # length.
    self.SendReply(
        rdf_client.BufferReference(
            offset=args.offset, length=len(data), data=digest))

Response: Reads a buffer on the client and sends it to the server.


Instruction:
Explain what the following Python 3 code does
def Run(self, args):
    # Make sure we limit the size of our output
    if args.length > constants.CLIENT_MAX_BUFFER_SIZE:
      raise RuntimeError("Can not read buffers this large.")

    data = vfs.ReadVFS(args.pathspec, args.offset, args.length)

    digest = hashlib.sha256(data).digest()

    # Now report the hash of this blob to our flow as well as the offset and
    # length.
    self.SendReply(
        rdf_client.BufferReference(
            offset=args.offset, length=len(data), data=digest))

Response: Reads a buffer on the client and sends it to the server.


Instruction:
Can you write a function in Python 3 where it
writes the blob to a file and returns its path.

Response:
def WriteBlobToFile(self, request):
    """Writes the blob to a file and returns its path."""
    # First chunk truncates the file, later chunks append.
    if request.offset == 0:
      mode = "w+b"
    else:
      mode = "r+b"

    temp_file = tempfiles.CreateGRRTempFile(
        filename=request.write_path, mode=mode)
    with temp_file:
      path = temp_file.name
      temp_file.seek(0, 2)
      if temp_file.tell() != request.offset:
        raise IOError("Chunks out of order Error.")

      # Write the new chunk.
      temp_file.write(request.executable.data)

    return path


Instruction:
Can you tell what is the following Python 3 function doing
def CleanUp(self, path):
    try:
      if os.path.exists(path):
        os.remove(path)
    except (OSError, IOError) as e:
      logging.info("Failed to remove temporary file %s. Err: %s", path, e)

Response: Removes the temp file.


Instruction:
Make a summary of the following Python 3 code
def Run(self, args):
    # Verify the executable blob.
    args.executable.Verify(
        config.CONFIG["Client.executable_signing_public_key"])

    path = self.WriteBlobToFile(args)

    # Only actually run the file on the last chunk.
    if not args.more_data:
      self.ProcessFile(path, args)
      self.CleanUp(path)

Response: Write the blob to disk and process the file.


Instruction:
Here you have a function in Python 3, explain what it does
def Run(self, args):
    time_start = time.time()

    args.python_code.Verify(
        config.CONFIG["Client.executable_signing_public_key"])

    # The execed code can assign to this variable if it wants to return data.
    logging.debug("exec for python code %s", args.python_code.data[0:100])

    context = globals().copy()
    context["py_args"] = args.py_args.ToDict()
    context["magic_return_str"] = ""
    # Export the Progress function to allow python hacks to call it.
    context["Progress"] = self.Progress

    stdout = io.StringIO()
    with utils.Stubber(sys, "stdout", StdOutHook(stdout)):
      exec(args.python_code.data, context)  # pylint: disable=exec-used

    stdout_output = stdout.getvalue()
    magic_str_output = context.get("magic_return_str")

    if stdout_output and magic_str_output:
      output = "Stdout: %s\nMagic Str:%s\n" % (stdout_output, magic_str_output)
    else:
      output = stdout_output or magic_str_output

    time_used = time.time() - time_start
    # We have to return microseconds.
    self.SendReply(
        rdf_client_action.ExecutePythonResponse(
            time_used=int(1e6 * time_used), return_val=output))

Response: Runs the python code.


Instruction:
Implement a function in Python 3 to
return an iterator over the readable regions for this process.

Response:
def Regions(self,
              skip_mapped_files=False,
              skip_shared_regions=False,
              skip_executable_regions=False,
              skip_readonly_regions=False):
    """Returns an iterator over the readable regions for this process."""
    try:
      maps_file = open("/proc/" + str(self.pid) + "/maps", "r")
    except OSError as e:
      raise process_error.ProcessError(e)

    with maps_file:
      for line in maps_file:
        m = self.maps_re.match(line)
        if not m:
          continue
        start = int(m.group(1), 16)
        end = int(m.group(2), 16)
        region_protec = m.group(3)
        inode = int(m.group(6))

        if "r" in region_protec:
          if skip_mapped_files and inode != 0:
            continue
          if skip_shared_regions and "s" in region_protec:
            continue
          if skip_executable_regions and "x" in region_protec:
            continue
          if skip_readonly_regions and "w" not in region_protec:
            continue
          yield start, end - start


Instruction:
Explain what the following Python 3 code does
def CopyFileInZip(from_zip, from_name, to_zip, to_name=None, signer=None):
  data = from_zip.read(from_name)
  if to_name is None:
    to_name = from_name
  if signer:
    logging.debug("Signing %s", from_name)
    data = signer.SignBuffer(data)
  to_zip.writestr(to_name, data)

Response: Copy a file from a ZipFile and write it to a new ZipFile.


Instruction:
Given the following Python 3 function, write the documentation
def CreateNewZipWithSignedLibs(z_in,
                               z_out,
                               ignore_files=None,
                               signer=None,
                               skip_signing_files=None):
  """Copies files from one zip to another, signing all qualifying files."""
  ignore_files = ignore_files or []
  skip_signing_files = skip_signing_files or []
  extensions_to_sign = [".sys", ".exe", ".dll", ".pyd"]
  to_sign = []
  for template_file in z_in.namelist():
    if template_file not in ignore_files:
      extension = os.path.splitext(template_file)[1].lower()
      if (signer and template_file not in skip_signing_files and
          extension in extensions_to_sign):
        to_sign.append(template_file)
      else:
        CopyFileInZip(z_in, template_file, z_out)

  temp_files = {}
  for filename in to_sign:
    fd, path = tempfile.mkstemp()
    with os.fdopen(fd, "wb") as temp_fd:
      temp_fd.write(z_in.read(filename))
    temp_files[filename] = path

  try:
    signer.SignFiles(itervalues(temp_files))
  except AttributeError:
    for f in itervalues(temp_files):
      signer.SignFile(f)

  for filename, tempfile_path in iteritems(temp_files):
    z_out.writestr(filename, open(tempfile_path, "rb").read())

Response: Copies files from one zip to another with all signed libraries.


Instruction:
How would you code a function in Python 3 to
set the PE subsystem.

Response:
def SetPeSubsystem(fd, console=True):
  """Takes file like obj and returns (offset, value) for the PE subsystem."""
  current_pos = fd.tell()
  fd.seek(0x3c)  # _IMAGE_DOS_HEADER.e_lfanew
  header_offset = struct.unpack("<I", fd.read(4))[0]
  # _IMAGE_NT_HEADERS.OptionalHeader.Subsystem ( 0x18 + 0x44)
  subsystem_offset = header_offset + 0x5c
  fd.seek(subsystem_offset)
  if console:
    fd.write(b"\x03")
  else:
    fd.write(b"\x02")
  fd.seek(current_pos)


Instruction:
Can you write a function in Python 3 where it
generates a file from a template.

Response:
def GenerateFile(self, input_filename=None, output_filename=None):
    """Generates a file from a template, interpolating config values."""
    if input_filename is None:
      input_filename = output_filename + ".in"
    if output_filename[-3:] == ".in":
      output_filename = output_filename[:-3]
    logging.debug("Generating file %s from %s", output_filename, input_filename)

    with io.open(input_filename, "r") as fd:
      data = fd.read()

    with io.open(output_filename, "w") as fd:
      fd.write(config.CONFIG.InterpolateValue(data, context=self.context))


Instruction:
How would you implement a function in Python 3 that
prepares the build directory.

Response:
def MakeBuildDirectory(self):
    """Prepares the build directory."""
    # Create the build directory and let pyinstaller loose on it.
    self.build_dir = config.CONFIG.Get(
        "PyInstaller.build_dir", context=self.context)
    self.work_path = config.CONFIG.Get(
        "PyInstaller.workpath_dir", context=self.context)

    self.CleanDirectory(self.build_dir)
    self.CleanDirectory(self.work_path)


Instruction:
Can you implement a function in Python 3 that
uses pyinstaller to build a client package.

Response:
def BuildWithPyInstaller(self):
    """Use pyinstaller to build a client package."""
    self.CleanDirectory(
        config.CONFIG.Get("PyInstaller.distpath", context=self.context))

    logging.info("Copying pyinstaller support files")
    self.spec_file = os.path.join(self.build_dir, "grr.spec")

    with open(self.spec_file, "wb") as fd:
      fd.write(config.CONFIG.Get("PyInstaller.spec", context=self.context))

    with open(os.path.join(self.build_dir, "version.txt"), "wb") as fd:
      fd.write(config.CONFIG.Get("PyInstaller.version", context=self.context))

    shutil.copy(
        src=config.CONFIG.Get("PyInstaller.icon_path", context=self.context),
        dst=os.path.join(self.build_dir, u"grr.ico"))

    # We expect the onedir output at this location.
    self.output_dir = os.path.join(
        config.CONFIG.Get("PyInstaller.distpath", context=self.context),
        "grr-client")

    # Pyinstaller doesn't handle unicode strings.
    args = [
        "--distpath",
        str(config.CONFIG.Get("PyInstaller.distpath", context=self.context)),
        "--workpath",
        str(
            config.CONFIG.Get("PyInstaller.workpath_dir",
                              context=self.context)),
        str(self.spec_file)
    ]
    logging.info("Running pyinstaller: %s", args)
    PyInstallerMain.run(pyi_args=[utils.SmartStr(x) for x in args])

    # Clear out some crud that pyinstaller includes.
    for path in ["tcl", "tk", "pytz"]:
      dir_path = os.path.join(self.output_dir, path)
      try:
        shutil.rmtree(dir_path)
      except OSError:
        logging.error("Unable to remove directory: %s", dir_path)

      try:
        os.mkdir(dir_path)
      except OSError:
        logging.error("Unable to create directory: %s", dir_path)

      file_path = os.path.join(dir_path, path)
      try:
        # Create an empty file so the directories get put in the installers.
        with open(file_path, "wb"):
          pass
      except IOError:
        logging.error("Unable to create file: %s", file_path)

    version_ini = version.VersionPath()
    shutil.copy(version_ini, os.path.join(self.output_dir, "version.ini"))

    with open(os.path.join(self.output_dir, "build.yaml"), "wb") as fd:
      self.WriteBuildYaml(fd)


Instruction:
Write a Python 3 function for
writing build. yaml to fd.

Response:
def WriteBuildYaml(self, fd, build_timestamp=True):
    """Write build spec to fd."""
    output = {
        "Client.build_environment":
            rdf_client.Uname.FromCurrentSystem().signature(),
        "Template.build_type":
            config.CONFIG.Get("ClientBuilder.build_type", context=self.context),
        "Template.version_major":
            config.CONFIG.Get("Source.version_major", context=self.context),
        "Template.version_minor":
            config.CONFIG.Get("Source.version_minor", context=self.context),
        "Template.version_revision":
            config.CONFIG.Get("Source.version_revision", context=self.context),
        "Template.version_release":
            config.CONFIG.Get("Source.version_release", context=self.context),
        "Template.arch":
            config.CONFIG.Get("Client.arch", context=self.context)
    }

    if build_timestamp:
      output["Client.build_time"] = rdfvalue.RDFDatetime.Now()
    else:
      self.REQUIRED_BUILD_YAML_KEYS.remove("Client.build_time")

    for key, value in iteritems(output):
      output[key] = str(value)

    output["Template.build_context"] = self.context

    output_keys = set(iterkeys(output))
    if output_keys != self.REQUIRED_BUILD_YAML_KEYS:
      raise RuntimeError("Bad build.yaml: expected %s, got %s" %
                         (self.REQUIRED_BUILD_YAML_KEYS, output_keys))
    fd.write(yaml.Dump(output).encode("utf-8"))


Instruction:
Can you write a function in Python 3 where it
creates the executable template.

Response:
def MakeExecutableTemplate(self, output_file=None):
    """Create the executable template.

    The client is built in two phases. First an executable template is created
    with the client binaries contained inside a zip file. Then the installation
    package is created by appending the SFX extractor to this template and
    writing a config file into the zip file.  This technique allows the
    client build to be carried out once on the supported platform (e.g.
    windows with MSVS), but the deployable installer can be build on any
    platform which supports python.  Subclasses for each OS do the actual
    work, we just make sure the output directory is set up correctly here.

    Args:
      output_file: string filename where we will write the template.
    """
    self.template_file = output_file or config.CONFIG.Get(
        "ClientBuilder.template_path", context=self.context)
    utils.EnsureDirExists(os.path.dirname(self.template_file))


Instruction:
Given the following Python 3 function, write the documentation
def MakeZip(self, input_dir, output_file):
    logging.info("Generating zip template file at %s", output_file)
    basename, _ = os.path.splitext(output_file)
    # TODO(user):pytype: incorrect make_archive() definition in typeshed.
    # pytype: disable=wrong-arg-types
    shutil.make_archive(
        basename, "zip", base_dir=".", root_dir=input_dir, verbose=True)

Response: Creates a ZIP archive of the files in the input directory.


Instruction:
Can you tell what is the following Python 3 function doing
def GetClientConfig(self, context, validate=True, deploy_timestamp=True):
    with utils.TempDirectory() as tmp_dir:
      # Make sure we write the file in yaml format.
      filename = os.path.join(
          tmp_dir,
          config.CONFIG.Get("ClientBuilder.config_filename", context=context))

      new_config = config.CONFIG.MakeNewConfig()
      new_config.Initialize(reset=True, data="")
      new_config.SetWriteBack(filename)

      # Only copy certain sections to the client. We enumerate all
      # defined options and then resolve those from the config in the
      # client's context. The result is the raw option as if the
      # client read our config file.
      client_context = context[:]
      while contexts.CLIENT_BUILD_CONTEXT in client_context:
        client_context.remove(contexts.CLIENT_BUILD_CONTEXT)
      for descriptor in sorted(config.CONFIG.type_infos, key=lambda x: x.name):
        if descriptor.name in self.SKIP_OPTION_LIST:
          continue

        if descriptor.section in self.CONFIG_SECTIONS:
          value = config.CONFIG.GetRaw(
              descriptor.name, context=client_context, default=None)

          if value is not None:
            logging.debug("Copying config option to client: %s",
                          descriptor.name)

            new_config.SetRaw(descriptor.name, value)

      if deploy_timestamp:
        deploy_time_string = str(rdfvalue.RDFDatetime.Now())
        new_config.Set("Client.deploy_time", deploy_time_string)
      new_config.Write()

      if validate:
        self.ValidateEndConfig(new_config)

      private_validator = config.CONFIG.Get(
          "ClientBuilder.private_config_validator_class", context=context)
      if private_validator:
        try:
          validator = config_validator_base.PrivateConfigValidator.classes[
              private_validator]()
        except KeyError:
          logging.error(
              "Couldn't find config validator class %s, "
              "you probably need to copy it into lib/local", private_validator)
          raise
        validator.ValidateEndConfig(new_config, self.context)

      return io.open(filename, "r").read()

Response: Generates the client config file for inclusion in deployable binaries.


Instruction:
How would you code a function in Python 3 to
validate the end of a client config.

Response:
def ValidateEndConfig(self, config_obj, errors_fatal=True):
    """Given a generated client config, attempt to check for common errors."""
    errors = []

    if not config.CONFIG["Client.fleetspeak_enabled"]:
      location = config_obj.Get("Client.server_urls", context=self.context)
      if not location:
        errors.append("Empty Client.server_urls")

      for url in location:
        if not url.startswith("http"):
          errors.append("Bad Client.server_urls specified %s" % url)

    key_data = config_obj.GetRaw(
        "Client.executable_signing_public_key",
        default=None,
        context=self.context)
    if key_data is None:
      errors.append("Missing Client.executable_signing_public_key.")
    elif not key_data.startswith("-----BEGIN PUBLIC"):
      errors.append(
          "Invalid Client.executable_signing_public_key: %s" % key_data)
    else:
      rsa_key = rdf_crypto.RSAPublicKey()
      rsa_key.ParseFromHumanReadable(key_data)

    if not config.CONFIG["Client.fleetspeak_enabled"]:
      certificate = config_obj.GetRaw(
          "CA.certificate", default=None, context=self.context)
      if certificate is None or not certificate.startswith("-----BEGIN CERTIF"):
        errors.append("CA certificate missing from config.")

    for bad_opt in ["Client.private_key"]:
      if config_obj.Get(bad_opt, context=self.context, default=""):
        errors.append("Client cert in conf, this should be empty at deployment"
                      " %s" % bad_opt)

    if errors_fatal and errors:
      for error in errors:
        logging.error("Build Config Error: %s", error)
      raise RuntimeError("Bad configuration generated. Terminating.")
    else:
      return errors


Instruction:
Write a Python 3 script to
window specific config validations.

Response:
def ValidateEndConfig(self, config_obj, errors_fatal=True):
    """Windows specific config validations."""
    errors = super(WindowsClientRepacker, self).ValidateEndConfig(
        config_obj, errors_fatal=errors_fatal)

    install_dir = config_obj["Client.install_path"]
    for path in config_obj["Client.tempdir_roots"]:
      if path.startswith("/"):
        errors.append(
            "Client.tempdir_root %s starts with /, probably has Unix path." %
            path)
      if not path.startswith(install_dir):
        errors.append(
            "Client.tempdir_root %s is not inside the install_dir %s, this is "
            "a security risk" % ((path, install_dir)))

    if config_obj.Get("Logging.path").startswith("/"):
      errors.append("Logging.path starts with /, probably has Unix path. %s" %
                    config_obj["Logging.path"])

    if "Windows\\" in config_obj.GetRaw("Logging.path"):
      errors.append("Windows in Logging.path, you probably want "
                    "%(WINDIR|env) instead")

    if not config_obj["Client.binary_name"].endswith(".exe"):
      errors.append("Missing .exe extension on binary_name %s" %
                    config_obj["Client.binary_name"])

    if not config_obj["Nanny.binary"].endswith(".exe"):
      errors.append("Missing .exe extension on nanny_binary")

    if errors_fatal and errors:
      for error in errors:
        logging.error("Build Config Error: %s", error)
      raise RuntimeError("Bad configuration generated. Terminating.")
    else:
      return errors


Instruction:
Write a Python 3 script to
repackage the template zip with the installer.

Response:
def MakeDeployableBinary(self, template_path, output_path):
    """Repackage the template zip with the installer."""
    context = self.context + ["Client Context"]

    zip_data = io.BytesIO()
    output_zip = zipfile.ZipFile(
        zip_data, mode="w", compression=zipfile.ZIP_DEFLATED)

    z_template = zipfile.ZipFile(open(template_path, "rb"))

    # Track which files we've copied already.
    completed_files = [
        "grr-client.exe", "GRRservice.exe", "dbg_grr-client.exe",
        "dbg_GRRservice.exe"
    ]

    # Change the name of the main binary to the configured name.
    client_bin_name = config.CONFIG.Get("Client.binary_name", context=context)

    console_build = config.CONFIG.Get("ClientBuilder.console", context=context)
    if console_build:
      client_filename = "dbg_grr-client.exe"
      service_filename = "dbg_GRRservice.exe"
    else:
      client_filename = "grr-client.exe"
      service_filename = "GRRservice.exe"

    bin_name = z_template.getinfo(client_filename)
    output_zip.writestr(client_bin_name, z_template.read(bin_name))

    CopyFileInZip(z_template, "grr-client.exe.manifest", output_zip,
                  "%s.manifest" % client_bin_name)
    completed_files.append("grr-client.exe.manifest")

    # Change the name of the service binary to the configured name.
    service_template = z_template.getinfo(service_filename)

    service_bin_name = config.CONFIG.Get(
        "Nanny.service_binary_name", context=context)
    output_zip.writestr(service_bin_name, z_template.read(service_template))

    if config.CONFIG["Client.fleetspeak_enabled"]:
      self._GenerateFleetspeakServiceConfig(output_zip)

    if self.signed_template:
      # If the template libs were already signed we can skip signing
      CreateNewZipWithSignedLibs(
          z_template, output_zip, ignore_files=completed_files)
    else:
      CreateNewZipWithSignedLibs(
          z_template,
          output_zip,
          ignore_files=completed_files,
          signer=self.signer)
    output_zip.close()

    return self.MakeSelfExtractingZip(zip_data.getvalue(), output_path)


Instruction:
Here you have a function in Python 3, explain what it does
def _ValidateFleetspeakServiceConfig(self, config_path):
    with open(config_path, "rb") as f:
      pool = descriptor_pool.DescriptorPool()
      pool.AddDescriptor(fs_config_pb2.Config.DESCRIPTOR)
      parsed_config = text_format.Parse(
          f.read(), fs_system_pb2.ClientServiceConfig(), descriptor_pool=pool)
      if parsed_config.factory != "Daemon":
        raise BuildError(
            "Fleetspeak config does not have the expected factory type.")
      daemon_cfg = fs_config_pb2.Config()
      parsed_config.config.Unpack(daemon_cfg)
      if not daemon_cfg.argv:
        raise BuildError(
            "Fleetspeak daemon service config does not specify command line "
            "args.")

Response: Validates a Fleetspeak service config.


Instruction:
Can you generate the documentation for the following Python 3 function
def MakeSelfExtractingZip(self, payload_data, output_path):
    context = self.context + ["Client Context"]

    src_zip = zipfile.ZipFile(io.BytesIO(payload_data), mode="r")
    zip_data = io.BytesIO()
    output_zip = zipfile.ZipFile(
        zip_data, mode="w", compression=zipfile.ZIP_DEFLATED)

    config_file_name = config.CONFIG.Get(
        "ClientBuilder.config_filename", context=context)
    # Copy the rest of the files from the package to the new zip.
    for template_file in src_zip.namelist():
      if template_file != config_file_name:
        # Avoid writing the config file twice if we're repacking a binary that
        # has already been run through deployment. We write it in the next step,
        # so no need to copy over from the original here.
        CopyFileInZip(src_zip, template_file, output_zip)

    client_config_content = self.GetClientConfig(context)

    output_zip.writestr(
        config_file_name,
        client_config_content.encode("utf-8"),
        compress_type=zipfile.ZIP_STORED)

    # The zip file comment is used by the self extractor to run the installation
    # script. Comment has to be `bytes` object because `zipfile` module is not
    # smart enough to properly handle `unicode` objects. We use the `encode`
    # method instead of `SmartStr` because we expect this option to be an
    # `unicode` object and in case it is not, we want it to blow up.
    output_zip.comment = b"$AUTORUN$>%s" % config.CONFIG.Get(
        "ClientBuilder.autorun_command_line", context=context).encode("utf-8")

    output_zip.close()

    utils.EnsureDirExists(os.path.dirname(output_path))
    with open(output_path, "wb") as fd:
      # First write the installer stub
      stub_data = io.BytesIO()
      unzipsfx_stub = config.CONFIG.Get(
          "ClientBuilder.unzipsfx_stub", context=context)
      stub_raw = open(unzipsfx_stub, "rb").read()

      # Check stub has been compiled with the requireAdministrator manifest.
      if b"level=\"requireAdministrator" not in stub_raw:
        raise RuntimeError("Bad unzip binary in use. Not compiled with the"
                           "requireAdministrator manifest option.")

      stub_data.write(stub_raw)

      # If in verbose mode, modify the unzip bins PE header to run in console
      # mode for easier debugging.
      SetPeSubsystem(
          stub_data,
          console=config.CONFIG.Get("ClientBuilder.console", context=context))

      # Now patch up the .rsrc section to contain the payload.
      end_of_file = zip_data.tell() + stub_data.tell()

      # This is the IMAGE_SECTION_HEADER.Name which is also the start of
      # IMAGE_SECTION_HEADER.
      offset_to_rsrc = stub_data.getvalue().find(b".rsrc")

      # IMAGE_SECTION_HEADER.PointerToRawData is a 32 bit int.
      stub_data.seek(offset_to_rsrc + 20)
      start_of_rsrc_section = struct.unpack("<I", stub_data.read(4))[0]

      # Adjust IMAGE_SECTION_HEADER.SizeOfRawData to span from the old start to
      # the end of file.
      stub_data.seek(offset_to_rsrc + 16)
      stub_data.write(struct.pack("<I", end_of_file - start_of_rsrc_section))

      # Concatenate stub and zip file.
      out_data = io.BytesIO()
      out_data.write(stub_data.getvalue())
      out_data.write(zip_data.getvalue())

      # Then write the actual output file.
      fd.write(out_data.getvalue())

    if self.signer:
      self.signer.SignFile(output_path)

    logging.info("Deployable binary generated at %s", output_path)

    return output_path

Response: Repack the installer into the payload.


Instruction:
Explain what the following Python 3 code does
def GenerateDPKGFiles(self, template_path):

    # Rename the generated binaries to the correct name.
    template_binary_dir = os.path.join(template_path, "dist/debian/grr-client")
    package_name = config.CONFIG.Get(
        "ClientBuilder.package_name", context=self.context)
    target_binary_dir = os.path.join(
        template_path, "dist/debian/%s%s" %
        (package_name,
         config.CONFIG.Get("ClientBuilder.target_dir", context=self.context)))
    if package_name == "grr-client":
      # Need to rename the template path or the move will fail.
      shutil.move(template_binary_dir, "%s-template" % template_binary_dir)
      template_binary_dir = "%s-template" % template_binary_dir

    utils.EnsureDirExists(os.path.dirname(target_binary_dir))
    shutil.move(template_binary_dir, target_binary_dir)

    shutil.move(
        os.path.join(target_binary_dir, "grr-client"),
        os.path.join(
            target_binary_dir,
            config.CONFIG.Get("Client.binary_name", context=self.context)))

    deb_in_dir = os.path.join(template_path, "dist/debian/debian.in/")

    self.GenerateDirectory(deb_in_dir, os.path.join(
        template_path, "dist/debian"), [("grr-client", package_name)])

    # Generate directories for the /usr/sbin link.
    utils.EnsureDirExists(
        os.path.join(template_path, "dist/debian/%s/usr/sbin" % package_name))

    if config.CONFIG.Get("Client.fleetspeak_enabled", context=self.context):
      self._GenerateFleetspeakConfig(template_path)
      shutil.rmtree(deb_in_dir)
      shutil.rmtree(os.path.join(template_path, "dist", "fleetspeak"))
      return

    # Generate the nanny template. This only exists from client version 3.1.2.5
    # onwards.
    if config.CONFIG["Template.version_numeric"] >= 3125:
      self.GenerateFile(
          os.path.join(target_binary_dir, "nanny.sh.in"),
          os.path.join(target_binary_dir, "nanny.sh"))

    # Generate the upstart template.
    self.GenerateFile(
        os.path.join(template_path, "dist/debian/upstart.in/grr-client.conf"),
        os.path.join(template_path, "dist/debian/%s.upstart" % package_name))

    # Generate the initd template. The init will not run if it detects upstart
    # is present.
    self.GenerateFile(
        os.path.join(template_path, "dist/debian/initd.in/grr-client"),
        os.path.join(template_path, "dist/debian/%s.init" % package_name))

    # Generate the systemd unit file.
    self.GenerateFile(
        os.path.join(template_path,
                     "dist/debian/systemd.in/grr-client.service"),
        os.path.join(template_path, "dist/debian/%s.service" % package_name))

    # Clean up the template dirs.
    shutil.rmtree(deb_in_dir)
    shutil.rmtree(os.path.join(template_path, "dist/debian/upstart.in"))
    shutil.rmtree(os.path.join(template_path, "dist/debian/initd.in"))
    shutil.rmtree(os.path.join(template_path, "dist/debian/systemd.in"))

Response: Generates the files needed by dpkg - buildpackage.


Instruction:
Can you tell what is the following Python 3 function doing
def MakeDeployableBinary(self, template_path, output_path):

    rpmbuild_binary = "/usr/bin/rpmbuild"
    if not os.path.exists(rpmbuild_binary):
      logging.error("rpmbuild not found, unable to repack client.")
      return

    with utils.TempDirectory() as tmp_dir:
      template_dir = os.path.join(tmp_dir, "dist")
      utils.EnsureDirExists(template_dir)

      zf = zipfile.ZipFile(template_path)
      for name in zf.namelist():
        dirname = os.path.dirname(name)
        utils.EnsureDirExists(os.path.join(template_dir, dirname))
        with open(os.path.join(template_dir, name), "wb") as fd:
          fd.write(zf.read(name))

      # Set up a RPM building environment.

      rpm_root_dir = os.path.join(tmp_dir, "rpmbuild")

      rpm_build_dir = os.path.join(rpm_root_dir, "BUILD")
      utils.EnsureDirExists(rpm_build_dir)

      rpm_buildroot_dir = os.path.join(rpm_root_dir, "BUILDROOT")
      utils.EnsureDirExists(rpm_buildroot_dir)

      rpm_rpms_dir = os.path.join(rpm_root_dir, "RPMS")
      utils.EnsureDirExists(rpm_rpms_dir)

      rpm_specs_dir = os.path.join(rpm_root_dir, "SPECS")
      utils.EnsureDirExists(rpm_specs_dir)

      template_binary_dir = os.path.join(tmp_dir, "dist/rpmbuild/grr-client")

      target_binary_dir = "%s%s" % (
          rpm_build_dir,
          config.CONFIG.Get("ClientBuilder.target_dir", context=self.context))

      utils.EnsureDirExists(os.path.dirname(target_binary_dir))
      try:
        shutil.rmtree(target_binary_dir)
      except OSError:
        pass
      # TODO(user):pytype: incorrect move() definition in typeshed.
      # pytype: disable=wrong-arg-types
      shutil.move(template_binary_dir, target_binary_dir)
      # pytype: enable=wrong-arg-types

      client_name = config.CONFIG.Get("Client.name", context=self.context)
      client_binary_name = config.CONFIG.Get(
          "Client.binary_name", context=self.context)
      if client_binary_name != "grr-client":
        # TODO(user):pytype: incorrect move() definition in typeshed.
        # pytype: disable=wrong-arg-types
        shutil.move(
            os.path.join(target_binary_dir, "grr-client"),
            os.path.join(target_binary_dir, client_binary_name))
        # pytype: enable=wrong-arg-types

      if config.CONFIG.Get("Client.fleetspeak_enabled", context=self.context):
        self._GenerateFleetspeakConfig(template_dir, rpm_build_dir)
        if not config.CONFIG.Get(
            "Client.fleetspeak_service_name", context=self.context):
          # The Fleetspeak service name is required when generating the RPM
          # spec file.
          raise BuildError("Client.fleetspeak_service_name is not set.")
      else:
        self._GenerateInitConfigs(template_dir, rpm_build_dir)

      # Generate spec
      spec_filename = os.path.join(rpm_specs_dir, "%s.spec" % client_name)
      self.GenerateFile(
          os.path.join(tmp_dir, "dist/rpmbuild/grr.spec.in"), spec_filename)

      # Generate prelinking blacklist file
      prelink_target_filename = os.path.join(rpm_build_dir,
                                             "etc/prelink.conf.d",
                                             "%s.conf" % client_name)

      utils.EnsureDirExists(os.path.dirname(prelink_target_filename))
      self.GenerateFile(
          os.path.join(tmp_dir, "dist/rpmbuild/prelink_blacklist.conf.in"),
          prelink_target_filename)

      # Create a client config.
      client_context = ["Client Context"] + self.context
      client_config_content = self.GetClientConfig(client_context)

      with open(
          os.path.join(
              target_binary_dir,
              config.CONFIG.Get(
                  "ClientBuilder.config_filename", context=self.context)),
          "wb") as fd:
        fd.write(client_config_content)

      # Set the daemon to executable.
      os.chmod(os.path.join(target_binary_dir, client_binary_name), 0o755)

      client_arch = config.CONFIG.Get("Template.arch", context=self.context)
      if client_arch == "amd64":
        client_arch = "x86_64"

      command = [
          rpmbuild_binary, "--define", "_topdir " + rpm_root_dir, "--target",
          client_arch, "--buildroot", rpm_buildroot_dir, "-bb", spec_filename
      ]
      try:
        subprocess.check_output(command, stderr=subprocess.STDOUT)
      except subprocess.CalledProcessError as e:
        logging.error("Error calling %s.", command)
        logging.error(e.output)
        raise

      client_version = config.CONFIG.Get(
          "Template.version_string", context=self.context)
      rpm_filename = os.path.join(
          rpm_rpms_dir, client_arch,
          "%s-%s-1.%s.rpm" % (client_name, client_version, client_arch))

      utils.EnsureDirExists(os.path.dirname(output_path))
      shutil.move(rpm_filename, output_path)

      logging.info("Created package %s", output_path)
      self.Sign(output_path)
      return output_path

Response: This method will create a. rpm file for the client.


Instruction:
Can you generate the documentation for the following Python 3 function
def _GenerateFleetspeakConfig(self, template_dir, rpm_build_dir):
    source_config = os.path.join(
        template_dir, "fleetspeak",
        os.path.basename(
            config.CONFIG.Get(
                "ClientBuilder.fleetspeak_config_path", context=self.context)))
    fleetspeak_service_dir = config.CONFIG.Get(
        "ClientBuilder.fleetspeak_service_dir", context=self.context)
    dest_config_dir = os.path.join(rpm_build_dir, fleetspeak_service_dir[1:])
    utils.EnsureDirExists(dest_config_dir)
    dest_config_path = os.path.join(
        dest_config_dir,
        config.CONFIG.Get(
            "Client.fleetspeak_unsigned_config_fname", context=self.context))
    self.GenerateFile(
        input_filename=source_config, output_filename=dest_config_path)

Response: Generates a Fleetspeak config for GRR.


Instruction:
Implement a function in Python 3 to
generate init - system configs.

Response:
def _GenerateInitConfigs(self, template_dir, rpm_build_dir):
    """Generates init-system configs."""
    client_name = config.CONFIG.Get("Client.name", context=self.context)
    initd_target_filename = os.path.join(rpm_build_dir, "etc/init.d",
                                         client_name)

    # Generate init.d
    utils.EnsureDirExists(os.path.dirname(initd_target_filename))
    self.GenerateFile(
        os.path.join(template_dir, "rpmbuild/grr-client.initd.in"),
        initd_target_filename)

    # Generate systemd unit
    if config.CONFIG["Template.version_numeric"] >= 3125:
      systemd_target_filename = os.path.join(rpm_build_dir,
                                             "usr/lib/systemd/system/",
                                             "%s.service" % client_name)

      utils.EnsureDirExists(os.path.dirname(systemd_target_filename))
      self.GenerateFile(
          os.path.join(template_dir, "rpmbuild/grr-client.service.in"),
          systemd_target_filename)


Instruction:
Can you write a function in Python 3 where it
builds necessary assets from sources.

Response:
def compile_protos():
  """Builds necessary assets from sources."""
  # If there's no makefile, we're likely installing from an sdist,
  # so there's no need to compile the protos (they should be already
  # compiled).
  if not os.path.exists(os.path.join(THIS_DIRECTORY, "makefile.py")):
    return

  # Only compile protobufs if we're inside GRR source tree.
  subprocess.check_call(
      ["python", "makefile.py", "--clean"], cwd=THIS_DIRECTORY)


Instruction:
Create a Python 3 function for
reading a cronjob from the database.

Response:
def ReadCronJobs(self, cronjob_ids=None):
    """Reads a cronjob from the database."""
    if cronjob_ids is None:
      res = [job.Copy() for job in itervalues(self.cronjobs)]

    else:
      res = []
      for job_id in cronjob_ids:
        try:
          res.append(self.cronjobs[job_id].Copy())
        except KeyError:
          raise db.UnknownCronJobError("Cron job with id %s not found." %
                                       job_id)

    for job in res:
      lease = self.cronjob_leases.get(job.cron_job_id)
      if lease:
        job.leased_until, job.leased_by = lease
    return res


Instruction:
Here you have a function in Python 3, explain what it does
def UpdateCronJob(self,
                    cronjob_id,
                    last_run_status=db.Database.unchanged,
                    last_run_time=db.Database.unchanged,
                    current_run_id=db.Database.unchanged,
                    state=db.Database.unchanged,
                    forced_run_requested=db.Database.unchanged):
    """Updates run information for an existing cron job."""
    job = self.cronjobs.get(cronjob_id)
    if job is None:
      raise db.UnknownCronJobError("Cron job %s not known." % cronjob_id)

    if last_run_status != db.Database.unchanged:
      job.last_run_status = last_run_status
    if last_run_time != db.Database.unchanged:
      job.last_run_time = last_run_time
    if current_run_id != db.Database.unchanged:
      job.current_run_id = current_run_id
    if state != db.Database.unchanged:
      job.state = state
    if forced_run_requested != db.Database.unchanged:
      job.forced_run_requested = forced_run_requested

Response: Updates the last_run_status current_run_id and state of an existing cronjob.


Instruction:
Write a Python 3 function that can
delete a cronjob along with all its runs.

Response:
def DeleteCronJob(self, cronjob_id):
    """Deletes a cronjob along with all its runs."""
    if cronjob_id not in self.cronjobs:
      raise db.UnknownCronJobError("Cron job %s not known." % cronjob_id)
    del self.cronjobs[cronjob_id]
    try:
      del self.cronjob_leases[cronjob_id]
    except KeyError:
      pass
    for job_run in self.ReadCronJobRuns(cronjob_id):
      del self.cronjob_runs[(cronjob_id, job_run.run_id)]


Instruction:
Explain what the following Python 3 code does
def LeaseCronJobs(self, cronjob_ids=None, lease_time=None):
    leased_jobs = []

    now = rdfvalue.RDFDatetime.Now()
    expiration_time = now + lease_time

    for job in itervalues(self.cronjobs):
      if cronjob_ids and job.cron_job_id not in cronjob_ids:
        continue
      existing_lease = self.cronjob_leases.get(job.cron_job_id)
      if existing_lease is None or existing_lease[0] < now:
        self.cronjob_leases[job.cron_job_id] = (expiration_time,
                                                utils.ProcessIdString())
        job = job.Copy()
        job.leased_until, job.leased_by = self.cronjob_leases[job.cron_job_id]
        leased_jobs.append(job)

    return leased_jobs

Response: Leases all available cron jobs.


Instruction:
Write a Python 3 script to
make leased cron jobs available for leasing again.

Response:
def ReturnLeasedCronJobs(self, jobs):
    """Makes leased cron jobs available for leasing again."""
    errored_jobs = []

    for returned_job in jobs:
      existing_lease = self.cronjob_leases.get(returned_job.cron_job_id)
      if existing_lease is None:
        errored_jobs.append(returned_job)
        continue

      if (returned_job.leased_until != existing_lease[0] or
          returned_job.leased_by != existing_lease[1]):
        errored_jobs.append(returned_job)
        continue

      del self.cronjob_leases[returned_job.cron_job_id]

    if errored_jobs:
      raise ValueError("Some jobs could not be returned: %s" %
                       ",".join(job.cron_job_id for job in errored_jobs))


Instruction:
Here you have a function in Python 3, explain what it does
def WriteCronJobRun(self, run_object):
    if run_object.cron_job_id not in self.cronjobs:
      raise db.UnknownCronJobError("Job with id %s not found." %
                                   run_object.cron_job_id)

    clone = run_object.Copy()
    clone.timestamp = rdfvalue.RDFDatetime.Now()
    self.cronjob_runs[(clone.cron_job_id, clone.run_id)] = clone

Response: Writes a cron job run object to the database.


Instruction:
Create a Python 3 function for
reading all cron job runs for a given job id.

Response:
def ReadCronJobRuns(self, job_id):
    """Reads all cron job runs for a given job id."""
    runs = [
        run for run in itervalues(self.cronjob_runs)
        if run.cron_job_id == job_id
    ]
    return sorted(runs, key=lambda run: run.started_at, reverse=True)


Instruction:
Can you create a Python 3 function that
reads a single cron job run from the db.

Response:
def ReadCronJobRun(self, job_id, run_id):
    """Reads a single cron job run from the db."""
    for run in itervalues(self.cronjob_runs):
      if run.cron_job_id == job_id and run.run_id == run_id:
        return run
    raise db.UnknownCronJobRunError(
        "Run with job id %s and run id %s not found." % (job_id, run_id))


Instruction:
Can you generate a brief explanation for the following Python 3 code
def DeleteOldCronJobRuns(self, cutoff_timestamp):
    deleted = 0
    for run in list(itervalues(self.cronjob_runs)):
      if run.timestamp < cutoff_timestamp:
        del self.cronjob_runs[(run.cron_job_id, run.run_id)]
        deleted += 1

    return deleted

Response: Deletes old cron job runs for a given job id.


Instruction:
How would you implement a function in Python 3 that
uploads a signed blob into the datastore.

Response:
def UploadSignedConfigBlob(content,
                           aff4_path,
                           client_context=None,
                           limit=None,
                           token=None):
  """Upload a signed blob into the datastore.

  Args:
    content: File content to upload.
    aff4_path: aff4 path to upload to.
    client_context: The configuration contexts to use.
    limit: The maximum size of the chunk to use.
    token: A security token.

  Raises:
    IOError: On failure to write.
  """
  if limit is None:
    limit = config.CONFIG["Datastore.maximum_blob_size"]

  # Get the values of these parameters which apply to the client running on the
  # target platform.
  if client_context is None:
    # Default to the windows client.
    client_context = ["Platform:Windows", "Client Context"]

  config.CONFIG.Validate(
      parameters="PrivateKeys.executable_signing_private_key")

  signing_key = config.CONFIG.Get(
      "PrivateKeys.executable_signing_private_key", context=client_context)

  verification_key = config.CONFIG.Get(
      "Client.executable_signing_public_key", context=client_context)

  signed_binary_utils.WriteSignedBinary(
      rdfvalue.RDFURN(aff4_path),
      content,
      signing_key,
      public_key=verification_key,
      chunk_size=limit,
      token=token)

  logging.info("Uploaded to %s", aff4_path)


Instruction:
Given the following Python 3 function, write the documentation
def _CheckAccess(self, username, subject_id, approval_type):
    precondition.AssertType(subject_id, Text)

    cache_key = (username, subject_id, approval_type)
    try:
      self.acl_cache.Get(cache_key)
      stats_collector_instance.Get().IncrementCounter(
          "approval_searches", fields=["-", "cache"])
      return True
    except KeyError:
      stats_collector_instance.Get().IncrementCounter(
          "approval_searches", fields=["-", "reldb"])

    approvals = data_store.REL_DB.ReadApprovalRequests(
        username, approval_type, subject_id=subject_id, include_expired=False)

    errors = []
    for approval in approvals:
      try:
        approval_checks.CheckApprovalRequest(approval)
        self.acl_cache.Put(cache_key, True)
        return
      except access_control.UnauthorizedAccess as e:
        errors.append(e)

    subject = approval_checks.BuildLegacySubject(subject_id, approval_type)
    if not errors:
      raise access_control.UnauthorizedAccess(
          "No approval found.", subject=subject)
    else:
      raise access_control.UnauthorizedAccess(
          " ".join(utils.SmartStr(e) for e in errors), subject=subject)

Response: Checks access to a given subject by a given user.


Instruction:
Can you write a function in Python 3 where it
checks whether a given user can access given client.

Response:
def CheckClientAccess(self, username, client_id):
    """Checks whether a given user can access given client."""
    self._CheckAccess(
        username, str(client_id),
        rdf_objects.ApprovalRequest.ApprovalType.APPROVAL_TYPE_CLIENT)


Instruction:
How would you explain what the following Python 3 function does
def CheckHuntAccess(self, username, hunt_id):

    self._CheckAccess(
        username, str(hunt_id),
        rdf_objects.ApprovalRequest.ApprovalType.APPROVAL_TYPE_HUNT)

Response: Checks whether a given user can access given hunt.


Instruction:
Given the following Python 3 function, write the documentation
def CheckCronJobAccess(self, username, cron_job_id):

    self._CheckAccess(
        username, str(cron_job_id),
        rdf_objects.ApprovalRequest.ApprovalType.APPROVAL_TYPE_CRON_JOB)

Response: Checks whether a given user can access given cron job.


Instruction:
Can you tell what is the following Python 3 function doing
def CheckIfCanStartClientFlow(self, username, flow_name):
    del username  # Unused.

    flow_cls = flow.GRRFlow.GetPlugin(flow_name)

    if not flow_cls.category:
      raise access_control.UnauthorizedAccess(
          "Flow %s can't be started via the API." % flow_name)

Response: Checks whether a given user can start a given flow.


Instruction:
Can you write a function in Python 3 where it
checks whether the user is an admin.

Response:
def CheckIfUserIsAdmin(self, username):
    """Checks whether the user is an admin."""

    user_obj = data_store.REL_DB.ReadGRRUser(username)
    if user_obj.user_type != user_obj.UserType.USER_TYPE_ADMIN:
      raise access_control.UnauthorizedAccess("User %s is not an admin." %
                                              username)


Instruction:
Can you tell what is the following Python 3 function doing
def Artifacts(self, os_name=None, cpe=None, label=None):
    hit = lambda x: x[0] == x[1] or not x[0]
    seq = [(self.os_name, os_name), (self.cpe, cpe), (self.label, label)]
    return all(map(hit, seq))

Response: Determines if the conditions applies to the host data.


Instruction:
Can you implement a function in Python 3 that
searches the condition for the specified values.

Response:
def Search(self, artifact, os_name=None, cpe=None, label=None):
    """Whether the condition contains the specified values.

    Args:
      artifact: A string identifier for the artifact.
      os_name: An OS string.
      cpe: A CPE string.
      label: A label string.

    Returns:
      True if the values match the non-empty query attributes.
      Empty query attributes are ignored in the comparison.
    """
    hit = lambda x: x[0] == x[1] or not x[0]
    seq = [(artifact, self.artifact), (os_name, self.os_name), (cpe, self.cpe),
           (label, self.label)]
    return all(map(hit, seq))


Instruction:
Write a Python 3 script to
register a callback for the given conditions.

Response:
def _Register(self, conditions, callback):
    """Map functions that should be called if the condition applies."""
    for condition in conditions:
      registered = self._registry.setdefault(condition, [])
      if callback and callback not in registered:
        registered.append(callback)


Instruction:
Can you tell what is the following Python 3 function doing
def Add(self, artifact=None, target=None, callback=None):
    # Cases where a target field is undefined or empty need special handling.
    # Repeated field helper in target yields results, so expand this out into a
    # list. If the attribute doesn't exist, default to an empty list.
    # Then, in either case, replace the empty list with one containing a single
    # None value.
    if target is None:
      target = Target()
    os_name = target.Get("os") or [None]
    cpe = target.Get("cpe") or [None]
    label = target.Get("label") or [None]
    attributes = itertools.product(os_name, cpe, label)
    new_conditions = [Condition(artifact, *attr) for attr in attributes]
    self.conditions.update(new_conditions)
    self._Register(new_conditions, callback)

Response: Adds a condition to the check list.


Instruction:
How would you explain what the following Python 3 function does
def Update(self, other, callback):
    self.conditions.update(other.conditions)
    self._Register(other.conditions, callback)

Response: Updates the set of existing triggers with the contents of the other set.


Instruction:
Can you write a function in Python 3 where it
tests if host data should trigger a check.

Response:
def Match(self, artifact=None, os_name=None, cpe=None, label=None):
    """Test if host data should trigger a check.

    Args:
      artifact: An artifact name.
      os_name: An OS string.
      cpe: A CPE string.
      label: A label string.

    Returns:
      A list of conditions that match.
    """
    return [
        c for c in self.conditions if c.Match(artifact, os_name, cpe, label)
    ]


Instruction:
How would you code a function in Python 3 to
search the host attributes that trigger data collection.

Response:
def Search(self, artifact=None, os_name=None, cpe=None, label=None):
    """Find the host attributes that trigger data collection.

    Args:
      artifact: An artifact name.
      os_name: An OS string.
      cpe: A CPE string.
      label: A label string.

    Returns:
      A list of conditions that contain the specified attributes.
    """
    return [
        c for c in self.conditions if c.Search(artifact, os_name, cpe, label)
    ]


Instruction:
Can you tell what is the following Python 3 function doing
def Artifacts(self, os_name=None, cpe=None, label=None):
    return [
        c.artifact for c in self.conditions if c.Artifacts(os_name, cpe, label)
    ]

Response: Returns a list of artifacts that correspond with other trigger conditions.


Instruction:
Write a Python 3 script to
find the methods that evaluate the data that meets the condition.

Response:
def Calls(self, conditions=None):
    """Find the methods that evaluate data that meets this condition.

    Args:
      conditions: A tuple of (artifact, os_name, cpe, label)

    Returns:
      A list of methods that evaluate the data.
    """
    results = set()
    if conditions is None:
      conditions = [None]
    for condition in conditions:
      for c in self.Match(*condition):
        results.update(self._registry.get(c, []))
    return results


Instruction:
Implement a function in Python 3 to
return plugins metadata in ApiReportDescriptor.

Response:
def GetReportDescriptor(cls):
    """Returns plugins' metadata in ApiReportDescriptor."""
    if cls.TYPE is None:
      raise ValueError("%s.TYPE is unintialized." % cls)

    if cls.TITLE is None:
      raise ValueError("%s.TITLE is unintialized." % cls)

    if cls.SUMMARY is None:
      raise ValueError("%s.SUMMARY is unintialized." % cls)

    return rdf_report_plugins.ApiReportDescriptor(
        type=cls.TYPE,
        name=cls.__name__,
        title=cls.TITLE,
        summary=cls.SUMMARY,
        requires_time_range=cls.REQUIRES_TIME_RANGE)


Instruction:
Can you implement a function in Python 3 that
makes a certificate and sign it with the CA s private key.

Response:
def MakeCASignedCert(common_name,
                     private_key,
                     ca_cert,
                     ca_private_key,
                     serial_number=2):
  """Make a cert and sign it with the CA's private key."""
  public_key = private_key.GetPublicKey()

  builder = x509.CertificateBuilder()

  builder = builder.issuer_name(ca_cert.GetIssuer())

  subject = x509.Name(
      [x509.NameAttribute(oid.NameOID.COMMON_NAME, common_name)])
  builder = builder.subject_name(subject)

  valid_from = rdfvalue.RDFDatetime.Now() - rdfvalue.Duration("1d")
  valid_until = rdfvalue.RDFDatetime.Now() + rdfvalue.Duration("3650d")
  builder = builder.not_valid_before(valid_from.AsDatetime())
  builder = builder.not_valid_after(valid_until.AsDatetime())

  builder = builder.serial_number(serial_number)
  builder = builder.public_key(public_key.GetRawPublicKey())

  builder = builder.add_extension(
      x509.BasicConstraints(ca=False, path_length=None), critical=True)
  certificate = builder.sign(
      private_key=ca_private_key.GetRawPrivateKey(),
      algorithm=hashes.SHA256(),
      backend=openssl.backend)
  return rdf_crypto.RDFX509Cert(certificate)


Instruction:
Can you create a Python 3 function that
generates a CA certificate.

Response:
def MakeCACert(private_key,
               common_name=u"grr",
               issuer_cn=u"grr_test",
               issuer_c=u"US"):
  """Generate a CA certificate.

  Args:
    private_key: The private key to use.
    common_name: Name for cert.
    issuer_cn: Name for issuer.
    issuer_c: Country for issuer.

  Returns:
    The certificate.
  """
  public_key = private_key.GetPublicKey()

  builder = x509.CertificateBuilder()

  issuer = x509.Name([
      x509.NameAttribute(oid.NameOID.COMMON_NAME, issuer_cn),
      x509.NameAttribute(oid.NameOID.COUNTRY_NAME, issuer_c)
  ])
  subject = x509.Name(
      [x509.NameAttribute(oid.NameOID.COMMON_NAME, common_name)])
  builder = builder.subject_name(subject)
  builder = builder.issuer_name(issuer)

  valid_from = rdfvalue.RDFDatetime.Now() - rdfvalue.Duration("1d")
  valid_until = rdfvalue.RDFDatetime.Now() + rdfvalue.Duration("3650d")
  builder = builder.not_valid_before(valid_from.AsDatetime())
  builder = builder.not_valid_after(valid_until.AsDatetime())

  builder = builder.serial_number(1)
  builder = builder.public_key(public_key.GetRawPublicKey())

  builder = builder.add_extension(
      x509.BasicConstraints(ca=True, path_length=None), critical=True)
  builder = builder.add_extension(
      x509.SubjectKeyIdentifier.from_public_key(public_key.GetRawPublicKey()),
      critical=False)
  certificate = builder.sign(
      private_key=private_key.GetRawPrivateKey(),
      algorithm=hashes.SHA256(),
      backend=openssl.backend)
  return rdf_crypto.RDFX509Cert(certificate)


Instruction:
Given the following Python 3 function, write the documentation
def _LoadAuditEvents(handlers,
                     get_report_args,
                     actions=None,
                     token=None,
                     transformers=None):
  """Returns AuditEvents for given handlers, actions, and timerange."""
  if transformers is None:
    transformers = {}

  if data_store.RelationalDBEnabled():
    entries = data_store.REL_DB.ReadAPIAuditEntries(
        min_timestamp=get_report_args.start_time,
        max_timestamp=get_report_args.start_time + get_report_args.duration,
        router_method_names=list(handlers.keys()))
    rows = [_EntryToEvent(entry, handlers, transformers) for entry in entries]
  else:
    entries = report_utils.GetAuditLogEntries(
        offset=get_report_args.duration,
        now=get_report_args.start_time + get_report_args.duration,
        token=token)
    if actions is None:
      actions = set(handlers.values())
    rows = [entry for entry in entries if entry.action in actions]
  rows.sort(key=lambda row: row.timestamp, reverse=True)
  return rows

Response: Loads audit events for given handlers actions and timerange.


Instruction:
Write a Python 3 function that can
convert an APIAuditEntry to an AuditEvent.

Response:
def _EntryToEvent(entry, handlers, transformers):
  """Converts an APIAuditEntry to a legacy AuditEvent."""
  event = rdf_events.AuditEvent(
      timestamp=entry.timestamp,
      user=entry.username,
      action=handlers[entry.router_method_name])

  for fn in transformers:
    fn(entry, event)

  return event


Instruction:
Write a Python 3 script to
extract a Client ID from an APIAuditEntry s HTTP request path.

Response:
def _ExtractClientIdFromPath(entry, event):
  """Extracts a Client ID from an APIAuditEntry's HTTP request path."""
  match = re.match(r".*(C\.[0-9a-fA-F]{16}).*", entry.http_request_path)
  if match:
    event.client = match.group(1)


Instruction:
How would you implement a function in Python 3 that
extracts a CronJob ID from an APIAuditEntry s HTTP request path.

Response:
def _ExtractCronJobIdFromPath(entry, event):
  """Extracts a CronJob ID from an APIAuditEntry's HTTP request path."""
  match = re.match(r".*cron-job/([^/]+).*", entry.http_request_path)
  if match:
    event.urn = "aff4:/cron/{}".format(match.group(1))


Instruction:
Write a Python 3 script for
extracting a Hunt ID from an APIAuditEntry s HTTP request path.

Response:
def _ExtractHuntIdFromPath(entry, event):
  """Extracts a Hunt ID from an APIAuditEntry's HTTP request path."""
  match = re.match(r".*hunt/([^/]+).*", entry.http_request_path)
  if match:
    event.urn = "aff4:/hunts/{}".format(match.group(1))


Instruction:
Create a Python 3 function for
returning the data for the given timerange.

Response:
def GetReportData(self, get_report_args, token=None):
    """Filter the cron job approvals in the given timerange."""
    ret = rdf_report_plugins.ApiReportData(
        representation_type=RepresentationType.AUDIT_CHART,
        audit_chart=rdf_report_plugins.ApiAuditChartReportData(
            used_fields=self.USED_FIELDS))

    ret.audit_chart.rows = _LoadAuditEvents(
        self.HANDLERS,
        get_report_args,
        transformers=[_ExtractClientIdFromPath],
        token=token)
    return ret


Instruction:
Write a Python 3 function that can
filter the last week of user actions.

Response:
def GetReportData(self, get_report_args, token):
    """Filter the last week of user actions."""
    ret = rdf_report_plugins.ApiReportData(
        representation_type=RepresentationType.PIE_CHART)

    counts = self._GetUserCounts(get_report_args, token)
    for username in aff4_users.GRRUser.SYSTEM_USERS:
      del counts[username]

    ret.pie_chart.data = [
        rdf_report_plugins.ApiReportDataPoint1D(x=count, label=user)
        for user, count in sorted(iteritems(counts))
    ]

    return ret


Instruction:
Here you have a function in Python 3, explain what it does
def GetReportData(self, get_report_args, token):
    ret = rdf_report_plugins.ApiReportData(
        representation_type=RepresentationType.STACK_CHART)

    week_duration = rdfvalue.Duration("7d")
    num_weeks = math.ceil(get_report_args.duration.seconds /
                          week_duration.seconds)
    weeks = range(0, num_weeks)
    start_time = get_report_args.start_time
    end_time = start_time + num_weeks * week_duration
    user_activity = collections.defaultdict(lambda: {week: 0 for week in weeks})

    entries = self._LoadUserActivity(
        start_time=get_report_args.start_time, end_time=end_time, token=token)

    for username, timestamp, count in entries:
      week = (timestamp - start_time).seconds // week_duration.seconds
      if week in user_activity[username]:
        user_activity[username][week] += count

    user_activity = sorted(iteritems(user_activity))
    user_activity = [(user, data)
                     for user, data in user_activity
                     if user not in aff4_users.GRRUser.SYSTEM_USERS]

    ret.stack_chart.data = [
        rdf_report_plugins.ApiReportDataSeries2D(
            label=user,
            points=(rdf_report_plugins.ApiReportDataPoint2D(x=x, y=y)
                    for x, y in sorted(data.items())))
        for user, data in user_activity
    ]

    return ret

Response: Returns the last 7 days of the user actions.


Instruction:
Can you tell what is the following Python 3 function doing
def _GetAFF4AttributeForReportType(report_type
                                  ):
  """Returns the corresponding AFF4 attribute for the given report type."""
  if report_type == rdf_stats.ClientGraphSeries.ReportType.GRR_VERSION:
    return aff4_stats.ClientFleetStats.SchemaCls.GRRVERSION_HISTOGRAM
  elif report_type == rdf_stats.ClientGraphSeries.ReportType.OS_TYPE:
    return aff4_stats.ClientFleetStats.SchemaCls.OS_HISTOGRAM
  elif report_type == rdf_stats.ClientGraphSeries.ReportType.OS_RELEASE:
    return aff4_stats.ClientFleetStats.SchemaCls.RELEASE_HISTOGRAM
  elif report_type == rdf_stats.ClientGraphSeries.ReportType.N_DAY_ACTIVE:
    return aff4_stats.ClientFleetStats.SchemaCls.LAST_CONTACTED_HISTOGRAM
  else:
    raise ValueError("Unknown report type %s." % report_type)

Response: Returns the corresponding AFF4 attribute for the given report type.


Instruction:
Create a Python 3 function to
write graph series for a particular client label to the DB.

Response:
def WriteGraphSeries(graph_series,
                     label,
                     token = None):
  """Writes graph series for a particular client label to the DB.

  Args:
    graph_series: A series of rdf_stats.Graphs containing aggregated data for a
      particular report-type.
    label: Client label by which data in the graph_series was aggregated.
    token: ACL token to use for writing to the legacy (non-relational)
      datastore.

  Raises:
    AFF4AttributeTypeError: If, when writing to the legacy DB, an unexpected
    report-data type is encountered.
  """
  if data_store.RelationalDBEnabled():
    data_store.REL_DB.WriteClientGraphSeries(graph_series, label)

  if _ShouldUseLegacyDatastore():
    # We need to use the __call__() method of the aff4.Attribute class
    # to instantiate Graph and GraphSeries objects, or AFF4Object.AddAttribute()
    # won't work.
    aff4_attr = _GetAFF4AttributeForReportType(graph_series.report_type)()

    if isinstance(aff4_attr, rdf_stats.GraphSeries):
      for graph in graph_series.graphs:
        aff4_attr.Append(graph)
    elif isinstance(aff4_attr, rdf_stats.Graph):
      for sample in graph_series.graphs[0]:
        aff4_attr.Append(x_value=sample.x_value, y_value=sample.y_value)
    else:
      raise AFF4AttributeTypeError(aff4_attr.__class__)

    with aff4.FACTORY.Create(
        GetAFF4ClientReportsURN().Add(label),
        aff4_type=aff4_stats.ClientFleetStats,
        mode="w",
        token=token) as stats_for_label:
      stats_for_label.AddAttribute(aff4_attr)


Instruction:
Create a Python 3 function to
fetch all graph series for the given label and report - type from the DB.

Response:
def FetchAllGraphSeries(
    label,
    report_type,
    period = None,
    token = None
):
  """Fetches graph series for the given label and report-type from the DB.

  Args:
    label: Client label to fetch data for.
    report_type: rdf_stats.ClientGraphSeries.ReportType to fetch data for.
    period: rdfvalue.Duration specifying how far back in time to fetch data. If
      not provided, all data for the given label and report-type will be
      returned.
    token: ACL token to use for reading from the legacy (non-relational)
      datastore.

  Raises:
    AFF4AttributeTypeError: If, when reading to the legacy DB, an unexpected
    report-data type is encountered.

  Returns:
    A dict mapping timestamps to graph-series. The timestamps
    represent when the graph-series were written to the datastore.
  """
  if _ShouldUseLegacyDatastore():
    return _FetchAllGraphSeriesFromTheLegacyDB(
        label, report_type, period=period, token=token)

  if period is None:
    time_range = None
  else:
    range_end = rdfvalue.RDFDatetime.Now()
    time_range = time_utils.TimeRange(range_end - period, range_end)
  return data_store.REL_DB.ReadAllClientGraphSeries(
      label, report_type, time_range=time_range)


Instruction:
Here you have a function in Python 3, explain what it does
def _FetchAllGraphSeriesFromTheLegacyDB(
    label,
    report_type,
    period = None,
    token = None
):
  """Fetches graph-series from the legacy DB [see FetchAllGraphSeries()]."""
  if period is None:
    time_range = aff4.ALL_TIMES
  else:
    range_end = rdfvalue.RDFDatetime.Now()
    time_range = (range_end - period, range_end)
  series_with_timestamps = {}
  try:
    stats_for_label = aff4.FACTORY.Open(
        GetAFF4ClientReportsURN().Add(label),
        aff4_type=aff4_stats.ClientFleetStats,
        mode="r",
        age=time_range,
        token=token)
  except aff4.InstantiationError:
    # Nothing to return for the given label and report-type.
    return series_with_timestamps
  aff4_attr = _GetAFF4AttributeForReportType(report_type)
  if aff4_attr.attribute_type == rdf_stats.GraphSeries:
    for graphs in stats_for_label.GetValuesForAttribute(aff4_attr):
      graph_series = rdf_stats.ClientGraphSeries(report_type=report_type)
      for graph in graphs:
        graph_series.graphs.Append(graph)
      series_with_timestamps[graphs.age] = graph_series
  elif aff4_attr.attribute_type == rdf_stats.Graph:
    for graph in stats_for_label.GetValuesForAttribute(aff4_attr):
      graph_series = rdf_stats.ClientGraphSeries(report_type=report_type)
      graph_series.graphs.Append(graph)
      series_with_timestamps[graph.age] = graph_series
  else:
    raise AFF4AttributeTypeError(aff4_attr.attribute_type)
  return series_with_timestamps

Response: Fetches all graph - series from the legacy DB [ see FetchAllGraphSeries() ]


Instruction:
Can you generate a brief explanation for the following Python 3 code
def FetchMostRecentGraphSeries(label,
                               report_type,
                               token = None
                              ):
  """Fetches the latest graph series for a client label from the DB.

  Args:
    label: Client label to fetch data for.
    report_type: rdf_stats.ClientGraphSeries.ReportType to fetch data for.
    token: ACL token to use for reading from the legacy (non-relational)
      datastore.

  Raises:
    AFF4AttributeTypeError: If, when reading to the legacy DB, an unexpected
    report-data type is encountered.

  Returns:
    The graph series for the given label and report type that was last
    written to the DB, or None if no series for that label and report-type
    exist.
  """
  if _ShouldUseLegacyDatastore():
    return _FetchMostRecentGraphSeriesFromTheLegacyDB(
        label, report_type, token=token)

  return data_store.REL_DB.ReadMostRecentClientGraphSeries(label, report_type)

Response: Fetches the latest graph series for a given label and report type from the DB.


Instruction:
Can you generate the documentation for the following Python 3 function
def _FetchMostRecentGraphSeriesFromTheLegacyDB(
    label,
    report_type,
    token = None
):
  """Fetches the latest graph-series for a client label from the legacy DB.

  Args:
    label: Client label to fetch data for.
    report_type: rdf_stats.ClientGraphSeries.ReportType to fetch data for.
    token: ACL token to use for reading from the DB.

  Raises:
    AFF4AttributeTypeError: If an unexpected report-data type is encountered.

  Returns:
    The graph series for the given label and report type that was last
    written to the DB, or None if no series for that label and report-type
    exist.
  """
  try:
    stats_for_label = aff4.FACTORY.Open(
        GetAFF4ClientReportsURN().Add(label),
        aff4_type=aff4_stats.ClientFleetStats,
        mode="r",
        token=token)
  except aff4.InstantiationError:
    # Nothing to return for the given label and report-type.
    return None
  aff4_attr = _GetAFF4AttributeForReportType(report_type)
  graph_series = rdf_stats.ClientGraphSeries(report_type=report_type)
  if aff4_attr.attribute_type == rdf_stats.GraphSeries:
    graphs = stats_for_label.Get(aff4_attr)
    if graphs is None:
      return None
    for graph in graphs:
      graph_series.graphs.Append(graph)
  elif aff4_attr.attribute_type == rdf_stats.Graph:
    graph = stats_for_label.Get(aff4_attr)
    if graph is None:
      return None
    graph_series.graphs.Append(graph)
  else:
    raise AFF4AttributeTypeError(aff4_attr.attribute_type)
  return graph_series

Response: Fetches the latest graph - series for a given label and report - type from the legacy DB.


Instruction:
Create a Python 3 function to
generate a new unique task_id.

Response:
def GenerateTaskID(self):
    """Generates a new, unique task_id."""
    # Random number can not be zero since next_id_base must increment.
    random_number = random.PositiveUInt16()

    # 16 bit random numbers
    with GrrMessage.lock:
      next_id_base = GrrMessage.next_id_base

      id_base = (next_id_base + random_number) & 0xFFFFFFFF
      if id_base < next_id_base:
        time.sleep(0.001)

      GrrMessage.next_id_base = id_base

    # 32 bit timestamp (in 1/1000 second resolution)
    time_base = (int(time.time() * 1000) & 0x1FFFFFFF) << 32

    task_id = time_base | id_base

    self.Set("task_id", task_id)
    return task_id


Instruction:
Here you have a function in Python 3, explain what it does
def AddFilesWithUnknownHashes(
    client_path_blob_refs,
    use_external_stores = True
):
  """Adds new files consisting of given blob references.

  Args:
    client_path_blob_refs: A dictionary mapping `db.ClientPath` instances to
      lists of blob references.
    use_external_stores: A flag indicating if the files should also be added to
      external file stores.

  Returns:
    A dictionary mapping `db.ClientPath` to hash ids of the file.

  Raises:
    BlobNotFoundError: If one of the referenced blobs cannot be found.
  """
  hash_id_blob_refs = dict()
  client_path_hash_id = dict()
  metadatas = dict()

  all_client_path_blob_refs = list()
  for client_path, blob_refs in iteritems(client_path_blob_refs):
    # In the special case where there is only one blob, we don't need to go to
    # the data store to read said blob and rehash it, we have all that
    # information already available. For empty files without blobs, we can just
    # hash the empty string instead.
    if len(blob_refs) <= 1:
      if blob_refs:
        hash_id = rdf_objects.SHA256HashID.FromBytes(
            blob_refs[0].blob_id.AsBytes())
      else:
        hash_id = rdf_objects.SHA256HashID.FromData(b"")

      client_path_hash_id[client_path] = hash_id
      hash_id_blob_refs[hash_id] = blob_refs
      metadatas[hash_id] = FileMetadata(
          client_path=client_path, blob_refs=blob_refs)
    else:
      for blob_ref in blob_refs:
        all_client_path_blob_refs.append((client_path, blob_ref))

  client_path_offset = collections.defaultdict(lambda: 0)
  client_path_sha256 = collections.defaultdict(hashlib.sha256)
  verified_client_path_blob_refs = collections.defaultdict(list)

  client_path_blob_ref_batches = collection.Batch(
      items=all_client_path_blob_refs, size=_BLOBS_READ_BATCH_SIZE)

  for client_path_blob_ref_batch in client_path_blob_ref_batches:
    blob_id_batch = set(
        blob_ref.blob_id for _, blob_ref in client_path_blob_ref_batch)
    blobs = data_store.BLOBS.ReadBlobs(blob_id_batch)

    for client_path, blob_ref in client_path_blob_ref_batch:
      blob = blobs[blob_ref.blob_id]
      if blob is None:
        message = "Could not find one of referenced blobs: {}".format(
            blob_ref.blob_id)
        raise BlobNotFoundError(message)

      offset = client_path_offset[client_path]
      if blob_ref.size != len(blob):
        raise ValueError(
            "Got conflicting size information for blob %s: %d vs %d." %
            (blob_ref.blob_id, blob_ref.size, len(blob)))
      if blob_ref.offset != offset:
        raise ValueError(
            "Got conflicting offset information for blob %s: %d vs %d." %
            (blob_ref.blob_id, blob_ref.offset, offset))

      verified_client_path_blob_refs[client_path].append(blob_ref)
      client_path_offset[client_path] = offset + len(blob)
      client_path_sha256[client_path].update(blob)

  for client_path in iterkeys(client_path_sha256):
    sha256 = client_path_sha256[client_path].digest()
    hash_id = rdf_objects.SHA256HashID.FromBytes(sha256)

    client_path_hash_id[client_path] = hash_id
    hash_id_blob_refs[hash_id] = verified_client_path_blob_refs[client_path]

  data_store.REL_DB.WriteHashBlobReferences(hash_id_blob_refs)

  if use_external_stores:
    for client_path in iterkeys(verified_client_path_blob_refs):
      metadatas[client_path_hash_id[client_path]] = FileMetadata(
          client_path=client_path,
          blob_refs=verified_client_path_blob_refs[client_path])

    EXTERNAL_FILE_STORE.AddFiles(metadatas)

  return client_path_hash_id

Response: Adds new files consisting of given blobs.


Instruction:
Here you have a function in Python 3, explain what it does
def AddFileWithUnknownHash(client_path, blob_refs, use_external_stores=True):
  precondition.AssertType(client_path, db.ClientPath)
  precondition.AssertIterableType(blob_refs, rdf_objects.BlobReference)
  return AddFilesWithUnknownHashes(
      {client_path: blob_refs},
      use_external_stores=use_external_stores)[client_path]

Response: Adds a new file consisting of given blob IDs.


Instruction:
Here you have a function in Python 3, explain what it does
def CheckHashes(hash_ids):
  return {
      k: bool(v)
      for k, v in data_store.REL_DB.ReadHashBlobReferences(hash_ids).items()
  }

Response: Checks if files with given hashes are present in the file store.


Instruction:
How would you code a function in Python 3 to
open the latest content of a given file for reading.

Response:
def OpenFile(client_path, max_timestamp=None):
  """Opens latest content of a given file for reading.

  Args:
    client_path: A db.ClientPath object describing path to a file.
    max_timestamp: If specified, will open the last collected version with a
      timestamp equal or lower than max_timestamp. If not specified, will simply
      open the latest version.

  Returns:
    A file like object with random access support.

  Raises:
    FileHasNoContentError: if the file was never collected.
    MissingBlobReferencesError: if one of the blobs was not found.
  """

  path_info = data_store.REL_DB.ReadLatestPathInfosWithHashBlobReferences(
      [client_path], max_timestamp=max_timestamp)[client_path]

  if path_info is None:
    raise FileHasNoContentError(client_path)

  hash_id = rdf_objects.SHA256HashID.FromBytes(
      path_info.hash_entry.sha256.AsBytes())
  blob_references = data_store.REL_DB.ReadHashBlobReferences([hash_id])[hash_id]

  if blob_references is None:
    raise MissingBlobReferencesError(
        "File hash was expected to have corresponding "
        "blob references, but they were not found: %r" % hash_id)

  return BlobStream(client_path, blob_references, hash_id)


Instruction:
Implement a function in Python 3 to
stream contents of given files.

Response:
def StreamFilesChunks(client_paths, max_timestamp=None, max_size=None):
  """Streams contents of given files.

  Args:
    client_paths: db.ClientPath objects describing paths to files.
    max_timestamp: If specified, then for every requested file will open the
      last collected version of the file with a timestamp equal or lower than
      max_timestamp. If not specified, will simply open a latest version for
      each file.
    max_size: If specified, only the chunks covering max_size bytes will be
      returned.

  Yields:
    StreamedFileChunk objects for every file read. Chunks will be returned
    sequentially, their order will correspond to the client_paths order.
    Files having no content will simply be ignored.
  """

  path_infos_by_cp = (
      data_store.REL_DB.ReadLatestPathInfosWithHashBlobReferences(
          client_paths, max_timestamp=max_timestamp))

  hash_ids_by_cp = {
      cp: rdf_objects.SHA256HashID.FromBytes(pi.hash_entry.sha256.AsBytes())
      for cp, pi in iteritems(path_infos_by_cp)
      if pi
  }

  blob_refs_by_hash_id = data_store.REL_DB.ReadHashBlobReferences(
      hash_ids_by_cp.values())

  all_chunks = []
  for cp in client_paths:
    try:
      hash_id = hash_ids_by_cp[cp]
    except KeyError:
      continue

    try:
      blob_refs = blob_refs_by_hash_id[hash_id]
    except KeyError:
      continue

    num_blobs = len(blob_refs)
    total_size = 0
    for ref in blob_refs:
      total_size += ref.size

    cur_size = 0
    for i, ref in enumerate(blob_refs):
      all_chunks.append((cp, ref.blob_id, i, num_blobs, ref.offset, total_size))

      cur_size += ref.size
      if max_size is not None and cur_size >= max_size:
        break

  for batch in collection.Batch(all_chunks, STREAM_CHUNKS_READ_AHEAD):
    blobs = data_store.BLOBS.ReadBlobs(
        [blob_id for cp, blob_id, i, num_blobs, offset, total_size in batch])
    for cp, blob_id, i, num_blobs, offset, total_size in batch:
      yield StreamedFileChunk(cp, blobs[blob_id], i, num_blobs, offset,
                              total_size)


Instruction:
How would you implement a function in Python 3 that
adds multiple files to the file store.

Response:
def AddFiles(self, hash_id_metadatas):
    """Adds multiple files to the file store.

    Args:
      hash_id_metadatas: A dictionary mapping hash ids to file metadata (a tuple
        of hash client path and blob references).
    """
    for hash_id, metadata in iteritems(hash_id_metadatas):
      self.AddFile(hash_id, metadata)


Instruction:
Implement a Python 3 function for
fetching a chunk corresponding to the current offset.

Response:
def _GetChunk(self):
    """Fetches a chunk corresponding to the current offset."""

    found_ref = None
    for ref in self._blob_refs:
      if self._offset >= ref.offset and self._offset < (ref.offset + ref.size):
        found_ref = ref
        break

    if not found_ref:
      return None, None

    # If self._current_ref == found_ref, then simply return previously found
    # chunk. Otherwise, update self._current_chunk value.
    if self._current_ref != found_ref:
      self._current_ref = found_ref

      data = data_store.BLOBS.ReadBlobs([found_ref.blob_id])
      self._current_chunk = data[found_ref.blob_id]

    return self._current_chunk, self._current_ref


Instruction:
How would you code a function in Python 3 to
read data from the server.

Response:
def Read(self, length=None):
    """Reads data."""

    if length is None:
      length = self._length - self._offset

    if length > self._max_unbound_read:
      raise OversizedReadError("Attempted to read %d bytes when "
                               "Server.max_unbound_read_size is %d" %
                               (length, self._max_unbound_read))

    result = io.BytesIO()
    while result.tell() < length:
      chunk, ref = self._GetChunk()
      if not chunk:
        break

      part = chunk[self._offset - ref.offset:]
      if not part:
        break

      result.write(part)
      self._offset += min(length, len(part))

    return result.getvalue()[:length]


Instruction:
Write a Python 3 function that can
move the reading cursor.

Response:
def Seek(self, offset, whence=os.SEEK_SET):
    """Moves the reading cursor."""

    if whence == os.SEEK_SET:
      self._offset = offset
    elif whence == os.SEEK_CUR:
      self._offset += offset
    elif whence == os.SEEK_END:
      self._offset = self._length + offset
    else:
      raise ValueError("Invalid whence argument: %s" % whence)


Instruction:
Create a Python 3 function for
uploading chunks of a file on a given path to the transfer store flow.

Response:
def UploadFilePath(self, filepath, offset=0, amount=None):
    """Uploads chunks of a file on a given path to the transfer store flow.

    Args:
      filepath: A path to the file to upload.
      offset: An integer offset at which the file upload should start on.
      amount: An upper bound on number of bytes to stream. If it is `None` then
        the whole file is uploaded.

    Returns:
      A `BlobImageDescriptor` object.
    """
    return self._UploadChunkStream(
        self._streamer.StreamFilePath(filepath, offset=offset, amount=amount))


Instruction:
How would you explain what the following Python 3 function does
def UploadFile(self, fd, offset=0, amount=None):
    return self._UploadChunkStream(
        self._streamer.StreamFile(fd, offset=offset, amount=amount))

Response: Uploads a chunk of a given file descriptor to the transfer store flow.


Instruction:
Implement a Python 3 function for
uploading a single chunk to the transfer store flow.

Response:
def _UploadChunk(self, chunk):
    """Uploads a single chunk to the transfer store flow.

    Args:
      chunk: A chunk to upload.

    Returns:
      A `BlobImageChunkDescriptor` object.
    """
    blob = _CompressedDataBlob(chunk)

    self._action.ChargeBytesToSession(len(chunk.data))
    self._action.SendReply(blob, session_id=self._TRANSFER_STORE_SESSION_ID)

    return rdf_client_fs.BlobImageChunkDescriptor(
        digest=hashlib.sha256(chunk.data).digest(),
        offset=chunk.offset,
        length=len(chunk.data))


Instruction:
Can you tell what is the following Python 3 function doing
def Parse(self, stat, file_object, knowledge_base):
    _ = knowledge_base
    entries = []

    crondata = file_object.read().decode("utf-8")
    jobs = crontab.CronTab(tab=crondata)

    for job in jobs:
      entries.append(
          rdf_cronjobs.CronTabEntry(
              minute=str(job.minute),
              hour=str(job.hour),
              dayofmonth=str(job.dom),
              month=str(job.month),
              dayofweek=str(job.dow),
              command=str(job.command),
              comment=str(job.comment)))

    try:
      source_urn = file_object.urn
    except AttributeError:
      source_urn = None

    yield rdf_cronjobs.CronTabFile(aff4path=source_urn, jobs=entries)

Response: Parse the crontab file.


Instruction:
Write a Python 3 script to
parse the file finder condition types into the condition objects.

Response:
def Parse(conditions):
    """Parses the file finder condition types into the condition objects.

    Args:
      conditions: An iterator over `FileFinderCondition` objects.

    Yields:
      `MetadataCondition` objects that correspond to the file-finder conditions.
    """
    kind = rdf_file_finder.FileFinderCondition.Type
    classes = {
        kind.MODIFICATION_TIME: ModificationTimeCondition,
        kind.ACCESS_TIME: AccessTimeCondition,
        kind.INODE_CHANGE_TIME: InodeChangeTimeCondition,
        kind.SIZE: SizeCondition,
        kind.EXT_FLAGS: ExtFlagsCondition,
    }

    for condition in conditions:
      try:
        yield classes[condition.condition_type](condition)
      except KeyError:
        pass


Instruction:
Given the following Python 3 function, write the documentation
def Parse(conditions):
    kind = rdf_file_finder.FileFinderCondition.Type
    classes = {
        kind.CONTENTS_LITERAL_MATCH: LiteralMatchCondition,
        kind.CONTENTS_REGEX_MATCH: RegexMatchCondition,
    }

    for condition in conditions:
      try:
        yield classes[condition.condition_type](condition)
      except KeyError:
        pass

Response: Parses the file finder condition types into the condition objects.


Instruction:
How would you explain what the following Python 3 function does
def Scan(self, fd,
           matcher):
    """Scans given file searching for occurrences of given pattern.

    Args:
      fd: A file descriptor of the file that needs to be searched.
      matcher: A matcher object specifying a pattern to search for.

    Yields:
      `BufferReference` objects pointing to file parts with matching content.
    """
    streamer = streaming.Streamer(
        chunk_size=self.CHUNK_SIZE, overlap_size=self.OVERLAP_SIZE)

    offset = self.params.start_offset
    amount = self.params.length
    for chunk in streamer.StreamFile(fd, offset=offset, amount=amount):
      for span in chunk.Scan(matcher):
        ctx_begin = max(span.begin - self.params.bytes_before, 0)
        ctx_end = min(span.end + self.params.bytes_after, len(chunk.data))
        ctx_data = chunk.data[ctx_begin:ctx_end]

        yield rdf_client.BufferReference(
            offset=chunk.offset + ctx_begin,
            length=len(ctx_data),
            data=ctx_data)

        if self.params.mode == self.params.Mode.FIRST_HIT:
          return

Response: Scans given file for occurrences of given pattern.


Instruction:
Write a Python 3 function for
initializing the server logging configuration.

Response:
def ServerLoggingStartupInit():
  """Initialize the server logging configuration."""
  global LOGGER
  if local_log:
    logging.debug("Using local LogInit from %s", local_log)
    local_log.LogInit()
    logging.debug("Using local AppLogInit from %s", local_log)
    LOGGER = local_log.AppLogInit()
  else:
    LogInit()
    LOGGER = AppLogInit()


Instruction:
Here you have a function in Python 3, explain what it does
def GetNewEventId(self, event_time=None):
    if event_time is None:
      event_time = int(time.time() * 1e6)

    return "%s:%s:%s" % (event_time, socket.gethostname(), os.getpid())

Response: Return a unique Event ID string.


Instruction:
Explain what the following Python 3 code does
def LogHttpAdminUIAccess(self, request, response):
    # TODO(user): generate event_id elsewhere and use it for all the log
    # messages that have to do with handling corresponding request.
    event_id = self.GetNewEventId()

    api_method = response.headers.get("X-API-Method", "unknown")
    api_reason = response.headers.get("X-GRR-Reason", "none")
    log_msg = "%s API call [%s] by %s (reason: %s): %s [%d]" % (
        event_id, api_method, request.user, api_reason, request.full_path,
        response.status_code)
    logging.info(log_msg)

    if response.headers.get("X-No-Log") != "True":
      if data_store.RelationalDBEnabled():
        entry = rdf_objects.APIAuditEntry.FromHttpRequestResponse(
            request, response)
        data_store.REL_DB.WriteAPIAuditEntry(entry)

Response: Logs an http based API call.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def LogHttpFrontendAccess(self, request, source=None, message_count=None):
    # TODO(user): generate event_id elsewhere and use it for all the log
    # messages that have to do with handling corresponding request.
    event_id = self.GetNewEventId()

    log_msg = "%s-%s [%s]: %s %s %s %s (%d)" % (
        event_id, request.source_ip, source or "<unknown>", request.method,
        request.url, request.user_agent, request.user, message_count or 0)
    logging.info(log_msg)

Response: Writes a log entry for a Frontend or UI Request.


Instruction:
Here you have a function in Python 3, explain what it does
def _GetPkgResources(package_name, filepath):
  requirement = pkg_resources.Requirement.parse(package_name)
  try:
    return pkg_resources.resource_filename(requirement, filepath)
  except pkg_resources.DistributionNotFound:
    # It may be that the working set is not in sync (e.g. if sys.path was
    # manipulated). Try to reload it just in case.
    pkg_resources.working_set = pkg_resources.WorkingSet()
    try:
      return pkg_resources.resource_filename(requirement, filepath)
    except pkg_resources.DistributionNotFound:
      logging.error("Distribution %s not found. Is it installed?", package_name)
      return None

Response: A wrapper for the pkg_resources. resource_filename function.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def ResourcePath(package_name, filepath):
  # If we are running a pyinstaller-built binary we rely on the sys.prefix
  # code below and avoid running this which will generate confusing error
  # messages.
  if not getattr(sys, "frozen", None):
    target = _GetPkgResources(package_name, filepath)
    if target and os.access(target, os.R_OK):
      return target

  # Installing from wheel places data_files relative to sys.prefix and not
  # site-packages. If we can not find in site-packages, check sys.prefix
  # instead.
  # https://python-packaging-user-guide.readthedocs.io/en/latest/distributing/#data-files
  target = os.path.join(sys.prefix, filepath)
  if target and os.access(target, os.R_OK):
    return target

  return None

Response: Computes a path to the specified resource in the specified package.


Instruction:
Create a Python 3 function for
computing a path to the specified module.

Response:
def ModulePath(module_name):
  """Computes a path to the specified module.

  Args:
    module_name: A name of the module to get the path for.

  Returns:
    A path to the specified module.

  Raises:
    ImportError: If specified module cannot be imported.
  """
  module = importlib.import_module(module_name)
  path = inspect.getfile(module)
  # TODO: In Python 2 `inspect.getfile` returns a byte string, so
  # we have to decode that in order to be consistent with Python 3.
  if compatibility.PY2:
    path = path.decode("utf-8")

  # In case of modules with want a path to the directory rather than to the
  # `__init__.py` file itself.
  if os.path.basename(path).startswith("__init__."):
    path = os.path.dirname(path)

  # Sometimes __file__ points at a .pyc file, when we really mean the .py.
  if path.endswith(".pyc"):
    path = path[:-4] + ".py"

  return path


Instruction:
Here you have a function in Python 3, explain what it does
def WithLimitedCallFrequency(min_time_between_calls):

  def Decorated(f):
    """Actual decorator implementation."""

    lock = threading.RLock()
    prev_times = {}
    prev_results = {}
    result_locks = {}

    @functools.wraps(f)
    def Fn(*args, **kwargs):
      """Wrapper around the decorated function."""

      if WITH_LIMITED_CALL_FREQUENCY_PASS_THROUGH:
        # This effectively turns off the caching.
        min_time = rdfvalue.Duration(0)
      else:
        min_time = min_time_between_calls

      key = (args, tuple(sorted(kwargs.items())))
      now = rdfvalue.RDFDatetime.Now()

      with lock:
        for k, prev_time in list(prev_times.items()):
          if now - prev_time >= min_time:
            prev_times.pop(k)
            prev_results.pop(k, None)
            result_locks.pop(k, None)

        try:
          # We eliminated all the old entries, so if the key is present
          # in the cache, it means that the data is fresh enough to be used.
          prev_time = prev_times[key]
          return prev_results[key]
        except KeyError:
          prev_time = None
          should_call = True

        if not should_call:
          return prev_results[key]

        try:
          result_lock = result_locks[key]
        except KeyError:
          result_lock = threading.RLock()
          result_locks[key] = result_lock

      with result_lock:
        t = prev_times.get(key)

        if t == prev_time:
          result = f(*args, **kwargs)
          with lock:
            prev_times[key] = rdfvalue.RDFDatetime.Now()
            prev_results[key] = result

          return result
        else:
          return prev_results[key]

    return Fn

  return Decorated

Response: A Python function decorator that returns a cached version of the passed - in function f.


Instruction:
Implement a function in Python 3 to
convert tuple field - name type to MetricFieldDefinition protos.

Response:
def FieldDefinitionProtosFromTuples(field_def_tuples):
  """Converts (field-name, type) tuples to MetricFieldDefinition protos."""
  # TODO: This needs fixing for Python 3.
  field_def_protos = []
  for field_name, field_type in field_def_tuples:
    if field_type in (int, long):
      field_type = rdf_stats.MetricFieldDefinition.FieldType.INT
    elif issubclass(field_type, Text):
      field_type = rdf_stats.MetricFieldDefinition.FieldType.STR
    else:
      raise ValueError("Invalid field type: %s" % field_type)
    field_def_protos.append(
        rdf_stats.MetricFieldDefinition(
            field_name=field_name, field_type=field_type))
  return field_def_protos


Instruction:
How would you code a function in Python 3 to
convert a list of MetricFieldDefinition protos to a list of tuples.

Response:
def FieldDefinitionTuplesFromProtos(field_def_protos):
  """Converts MetricFieldDefinition protos to (field-name, type) tuples."""
  # TODO: This needs fixing for Python 3.
  field_def_tuples = []
  for proto in field_def_protos:
    if proto.field_type == rdf_stats.MetricFieldDefinition.FieldType.INT:
      field_type = int
    elif proto.field_type == rdf_stats.MetricFieldDefinition.FieldType.STR:
      # Use old style str in Python 2 here or the streamz library will break.
      field_type = compatibility.builtins.str
    else:
      raise ValueError("Unknown field type: %s" % proto.field_type)
    field_def_tuples.append((proto.field_name, field_type))
  return field_def_tuples


Instruction:
How would you code a function in Python 3 to
convert Python types to MetricMetadata. ValueType enum values.

Response:
def MetricValueTypeFromPythonType(python_type):
  """Converts Python types to MetricMetadata.ValueType enum values."""
  if python_type in (int, long):
    return rdf_stats.MetricMetadata.ValueType.INT
  elif python_type == float:
    return rdf_stats.MetricMetadata.ValueType.FLOAT
  else:
    raise ValueError("Invalid value type: %s" % python_type)


Instruction:
Implement a function in Python 3 to
convert MetricMetadata. ValueType enums to corresponding Python types.

Response:
def PythonTypeFromMetricValueType(value_type):
  """Converts MetricMetadata.ValueType enums to corresponding Python types."""
  if value_type == rdf_stats.MetricMetadata.ValueType.INT:
    return int
  elif value_type == rdf_stats.MetricMetadata.ValueType.FLOAT:
    return float
  else:
    raise ValueError("Unknown value type: %s" % value_type)


Instruction:
Here you have a function in Python 3, explain what it does
def CreateCounterMetadata(metric_name, fields=None, docstring=None, units=None):
  return rdf_stats.MetricMetadata(
      varname=metric_name,
      metric_type=rdf_stats.MetricMetadata.MetricType.COUNTER,
      value_type=rdf_stats.MetricMetadata.ValueType.INT,
      fields_defs=FieldDefinitionProtosFromTuples(fields or []),
      docstring=docstring,
      units=units)

Response: Helper function for creating MetricMetadata for counter metrics.


Instruction:
Here you have a function in Python 3, explain what it does
def CreateGaugeMetadata(metric_name,
                        value_type,
                        fields=None,
                        docstring=None,
                        units=None):
  """Helper function for creating MetricMetadata for gauge metrics."""
  return rdf_stats.MetricMetadata(
      varname=metric_name,
      metric_type=rdf_stats.MetricMetadata.MetricType.GAUGE,
      value_type=MetricValueTypeFromPythonType(value_type),
      fields_defs=FieldDefinitionProtosFromTuples(fields or []),
      docstring=docstring,
      units=units)

Response: Helper function for creating MetricMetadata for gauge metrics.