Instruction:
Write a Python 3 function that can
import hashes from filename into store.

Response:
def ImportFile(store, filename, start):
  """Import hashes from 'filename' into 'store'."""
  with io.open(filename, "r") as fp:
    reader = csv.Reader(fp.read())
    i = 0
    current_row = None
    product_code_list = []
    op_system_code_list = []
    for row in reader:
      # Skip first row.
      i += 1
      if i and i % 5000 == 0:
        data_store.DB.Flush()
        print("Imported %d hashes" % i)
      if i > 1:
        if len(row) != 8:
          continue
        try:
          if i < start:
            continue
          if current_row:
            if current_row[0] == row[0]:
              # Same hash, add product/system
              product_code_list.append(int(row[5]))
              op_system_code_list.append(row[6])
              continue
            # Fall through and add current row.
          else:
            # First row.
            current_row = row
            product_code_list = [int(row[5])]
            op_system_code_list = [row[6]]
            continue
          _ImportRow(store, current_row, product_code_list, op_system_code_list)
          # Set new hash.
          current_row = row
          product_code_list = [int(row[5])]
          op_system_code_list = [row[6]]
        except Exception as e:  # pylint: disable=broad-except
          print("Failed at %d with %s" % (i, str(e)))
          return i - 1
    if current_row:
      _ImportRow(store, current_row, product_code_list, op_system_code_list)
    return i


Instruction:
How would you explain what the following Python 3 function does
def main(argv):
  del argv  # Unused.
  server_startup.Init()

  filename = flags.FLAGS.filename
  if not os.path.exists(filename):
    print("File %s does not exist" % filename)
    return

  with aff4.FACTORY.Create(
      filestore.NSRLFileStore.PATH,
      filestore.NSRLFileStore,
      mode="rw",
      token=aff4.FACTORY.root_token) as store:
    imported = ImportFile(store, filename, flags.FLAGS.start)
    data_store.DB.Flush()
    print("Imported %d hashes" % imported)

Response: Main function. Import a file into the file store.


Instruction:
Write a Python 3 script for
converting the canonical path as used by GRR to the local path.

Response:
def CanonicalPathToLocalPath(path):
  r"""Converts the canonical paths as used by GRR to OS specific paths.

  Due to the inconsistencies between handling paths in windows we need to
  convert a path to an OS specific version prior to using it. This function
  should be called just before any OS specific functions.

  Canonical paths on windows have:
    - / instead of \.
    - Begin with /X:// where X is the drive letter.

  Args:
    path: A canonical path specification.

  Returns:
    A windows specific path.
  """
  # Account for raw devices
  path = path.replace("/\\", "\\")
  path = path.replace("/", "\\")
  m = re.match(r"\\([a-zA-Z]):(.*)$", path)
  if m:
    path = "%s:\\%s" % (m.group(1), m.group(2).lstrip("\\"))

  return path


Instruction:
Can you generate the documentation for the following Python 3 function
def LocalPathToCanonicalPath(path):
  path_components = path.split("/")
  result = []
  for component in path_components:
    # Devices must maintain their \\ so they do not get broken up.
    m = re.match(r"\\\\.\\", component)

    # The component is not special and can be converted as normal
    if not m:
      component = component.replace("\\", "/")

    result.append(component)

  return utils.JoinPath(*result)

Response: Converts a local path from the local system s convention to the canonical path.


Instruction:
Write a Python 3 function for
providing chmod - like functionality for windows.

Response:
def WinChmod(filename, acl_list, user=None):
  """Provide chmod-like functionality for windows.

  Doco links:
    goo.gl/n7YR1
    goo.gl/rDv81
    goo.gl/hDobb

  Args:
    filename: target filename for acl

    acl_list: list of ntsecuritycon acl strings to be applied with bitwise OR.
              e.g. ["FILE_GENERIC_READ", "FILE_GENERIC_WRITE"]

    user: username string. If not specified we use the user we are running as.

  Raises:
    AttributeError: if a bad permission is passed
    RuntimeError: if filename doesn't exist
  """
  if user is None:
    user = win32api.GetUserName()

  if not os.path.exists(filename):
    raise RuntimeError("filename %s does not exist" % filename)

  acl_bitmask = 0
  for acl in acl_list:
    acl_bitmask |= getattr(ntsecuritycon, acl)

  dacl = win32security.ACL()
  win_user, _, _ = win32security.LookupAccountName("", user)

  dacl.AddAccessAllowedAce(win32security.ACL_REVISION, acl_bitmask, win_user)

  security_descriptor = win32security.GetFileSecurity(
      filename, win32security.DACL_SECURITY_INFORMATION)

  # Tell windows to set the acl and mark it as explicitly set
  security_descriptor.SetSecurityDescriptorDacl(DACL_PRESENT, dacl,
                                                DACL_DEFAULT)
  win32security.SetFileSecurity(
      filename, win32security.DACL_SECURITY_INFORMATION, security_descriptor)


Instruction:
How would you implement a function in Python 3 that
tries to find proxies by interrogating all the user's settings. This function is a modified urillib.getproxies_registry() from the standard library. We just store the proxy value in the environment for urllib to find it. TODO(user): Iterate through all the possible values if one proxy fails, in case more than one proxy is specified in different users profiles. Returns: A list of proxies.

Response:
def FindProxies():
  """Tries to find proxies by interrogating all the user's settings.

  This function is a modified urillib.getproxies_registry() from the
  standard library. We just store the proxy value in the environment
  for urllib to find it.

  TODO(user): Iterate through all the possible values if one proxy
  fails, in case more than one proxy is specified in different users
  profiles.

  Returns:
    A list of proxies.
  """

  proxies = []
  for i in range(0, 100):
    try:
      sid = winreg.EnumKey(winreg.HKEY_USERS, i)
    except OSError:
      break

    try:
      subkey = (
          sid + "\\Software\\Microsoft\\Windows"
          "\\CurrentVersion\\Internet Settings")

      internet_settings = winreg.OpenKey(winreg.HKEY_USERS, subkey)

      proxy_enable = winreg.QueryValueEx(internet_settings, "ProxyEnable")[0]

      if proxy_enable:
        # Returned as Unicode but problems if not converted to ASCII
        proxy_server = str(
            winreg.QueryValueEx(internet_settings, "ProxyServer")[0])
        if "=" in proxy_server:
          # Per-protocol settings
          for p in proxy_server.split(";"):
            protocol, address = p.split("=", 1)
            # See if address has a type:// prefix

            if not re.match("^([^/:]+)://", address):
              address = "%s://%s" % (protocol, address)

            proxies.append(address)
        else:
          # Use one setting for all protocols
          if proxy_server[:5] == "http:":
            proxies.append(proxy_server)
          else:
            proxies.append("http://%s" % proxy_server)

      internet_settings.Close()

    except (OSError, ValueError, TypeError):
      continue

  logging.debug("Found proxy servers: %s", proxies)

  return proxies


Instruction:
Given the following Python 3 function, write the documentation
def GetRawDevice(path):
  path = CanonicalPathToLocalPath(path)
  # Try to expand the shortened paths
  try:
    path = win32file.GetLongPathName(path)
  except pywintypes.error:
    pass

  try:
    mount_point = win32file.GetVolumePathName(path)
  except pywintypes.error as details:
    logging.info("path not found. %s", details)
    raise IOError("No mountpoint for path: %s" % path)

  if not path.startswith(mount_point):
    stripped_mp = mount_point.rstrip("\\")
    if not path.startswith(stripped_mp):
      raise IOError("path %s is not mounted under %s" % (path, mount_point))

  corrected_path = LocalPathToCanonicalPath(path[len(mount_point):])
  corrected_path = utils.NormalizePath(corrected_path)

  volume = win32file.GetVolumeNameForVolumeMountPoint(mount_point).rstrip("\\")
  volume = LocalPathToCanonicalPath(volume)

  # The pathspec for the raw volume
  result = rdf_paths.PathSpec(
      path=volume,
      pathtype=rdf_paths.PathSpec.PathType.OS,
      mount_point=mount_point.rstrip("\\"))

  return result, corrected_path

Response: Resolves the raw device that contains the path.


Instruction:
Given the following Python 3 function, write the documentation
def _GetServiceKey():
  global _service_key

  if _service_key is None:
    hive = getattr(winreg, config.CONFIG["Client.config_hive"])
    path = config.CONFIG["Client.config_key"]

    # Don't use winreg.KEY_WOW64_64KEY since it breaks on Windows 2000
    _service_key = winreg.CreateKeyEx(hive, path, 0, winreg.KEY_ALL_ACCESS)

  return _service_key

Response: Returns the service key.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def RtlGetVersion(os_version_info_struct):
  rc = ctypes.windll.Ntdll.RtlGetVersion(ctypes.byref(os_version_info_struct))
  if rc != 0:
    raise OSError("Getting Windows version failed.")

Response: Wraps the lowlevel RtlGetVersion routine.


Instruction:
Write a Python 3 function for
getting the kernel version as string eg. 5. 1. 2600.

Response:
def KernelVersion():
  """Gets the kernel version as string, eg. "5.1.2600".

  Returns:
    The kernel version, or "unknown" in the case of failure.
  """
  rtl_osversioninfoexw = RtlOSVersionInfoExw()
  try:
    RtlGetVersion(rtl_osversioninfoexw)
  except OSError:
    return "unknown"

  return "%d.%d.%d" % (rtl_osversioninfoexw.dwMajorVersion,
                       rtl_osversioninfoexw.dwMinorVersion,
                       rtl_osversioninfoexw.dwBuildNumber)


Instruction:
How would you code a function in Python 3 to
write a heartbeat to the registry.

Response:
def Heartbeat(self):
    """Writes a heartbeat to the registry."""
    service_key = _GetServiceKey()
    try:
      winreg.SetValueEx(service_key, "Nanny.heartbeat", 0, winreg.REG_DWORD,
                        int(time.time()))
    except OSError as e:
      logging.debug("Failed to heartbeat nanny at %s: %s", service_key, e)


Instruction:
Write a Python 3 function for
writing the message into the transaction log.

Response:
def Write(self, grr_message):
    """Write the message into the transaction log.

    Args:
      grr_message: A GrrMessage instance.
    """
    grr_message = grr_message.SerializeToString()
    try:
      winreg.SetValueEx(_GetServiceKey(), "Transaction", 0, winreg.REG_BINARY,
                        grr_message)
      self._synced = False
    except OSError:
      pass


Instruction:
How would you code a function in Python 3 to
return a GrrMessage instance from the transaction log or None.

Response:
def Get(self):
    """Return a GrrMessage instance from the transaction log or None."""
    try:
      value, reg_type = winreg.QueryValueEx(_GetServiceKey(), "Transaction")
    except OSError:
      return

    if reg_type != winreg.REG_BINARY:
      return

    try:
      return rdf_flows.GrrMessage.FromSerializedString(value)
    except message.Error:
      return


Instruction:
Can you tell what is the following Python 3 function doing
def FromPath(cls, path, follow_symlink = True):
    # Note that we do not add type assertion for `path` here. The reason is that
    # many of the existing system calls (e.g. `os.listdir`) return results as
    # bytestrings in Python 2. This is fine because it also means that they also
    # accept bytestring paths as arguments in Python 2 (e.g. `os.stat`). Having
    # consistent types in both versions is certainly desired but it might be too
    # much work for too little benefit.
    precondition.AssertType(follow_symlink, bool)

    if follow_symlink:
      stat_obj = os.stat(path)
    else:
      stat_obj = os.lstat(path)

    return cls(path=path, stat_obj=stat_obj)

Response: Creates a new stat object from a given path.


Instruction:
Create a Python 3 function for
fetching Linux extended file flags.

Response:
def _FetchLinuxFlags(self):
    """Fetches Linux extended file flags."""
    if platform.system() != "Linux":
      return 0

    # Since we open a file in the next step we do not want to open a symlink.
    # `lsattr` returns an error when trying to check flags of a symlink, so we
    # assume that symlinks cannot have them.
    if self.IsSymlink():
      return 0

    # Some files (e.g. sockets) cannot be opened. For these we do not really
    # care about extended flags (they should have none). `lsattr` does not seem
    # to support such cases anyway. It is also possible that a file has been
    # deleted (because this method is used lazily).
    try:
      fd = os.open(self._path, os.O_RDONLY)
    except (IOError, OSError):
      return 0

    try:
      # This import is Linux-specific.
      import fcntl  # pylint: disable=g-import-not-at-top
      # TODO: On Python 2.7.6 `array.array` accepts only byte
      # strings as an argument. On Python 2.7.12 and 2.7.13 unicodes are
      # supported as well. On Python 3, only unicode strings are supported. This
      # is why, as a temporary hack, we wrap the literal with `str` call that
      # will convert it to whatever is the default on given Python version. This
      # should be changed to raw literal once support for Python 2 is dropped.
      buf = array.array(compatibility.NativeStr("l"), [0])
      # TODO(user):pytype: incorrect type spec for fcntl.ioctl
      # pytype: disable=wrong-arg-types
      fcntl.ioctl(fd, self.FS_IOC_GETFLAGS, buf)
      # pytype: enable=wrong-arg-types
      return buf[0]
    except (IOError, OSError):
      # File system does not support extended attributes.
      return 0
    finally:
      os.close(fd)


Instruction:
Can you tell what is the following Python 3 function doing
def Get(self, path, follow_symlink = True):
    key = self._Key(path=path, follow_symlink=follow_symlink)
    try:
      return self._cache[key]
    except KeyError:
      value = Stat.FromPath(path, follow_symlink=follow_symlink)
      self._cache[key] = value

      # If we are not following symlinks and the file is a not symlink then
      # the stat result for this file stays the same even if we want to follow
      # symlinks.
      if not follow_symlink and not value.IsSymlink():
        self._cache[self._Key(path=path, follow_symlink=True)] = value

      return value

Response: Get a stat object corresponding to the given path.


Instruction:
Can you generate the documentation for the following Python 3 function
def GetClassesByArtifact(cls, artifact_name):
    return [
        cls.classes[c]
        for c in cls.classes
        if artifact_name in cls.classes[c].supported_artifacts
    ]

Response: Get the classes that support parsing a given artifact.


Instruction:
Create a Python 3 function to
enumerate all the configs in the given stats and file_objects.

Response:
def EnumerateAllConfigs(self, stats, file_objects):
    """Generate RDFs for the fully expanded configs.

    Args:
      stats: A list of RDF StatEntries corresponding to the file_objects.
      file_objects: A list of file handles.

    Returns:
      A tuple of a list of RDFValue PamConfigEntries found & a list of strings
      which are the external config references found.
    """
    # Convert the stats & file_objects into a cache of a
    # simple path keyed dict of file contents.
    cache = {}
    for stat_obj, file_obj in zip(stats, file_objects):
      cache[stat_obj.pathspec.path] = utils.ReadFileBytesAsUnicode(file_obj)

    result = []
    external = []
    # Check to see if we have the old pam config file laying around.
    if self.OLD_PAMCONF_FILENAME in cache:
      # The PAM documentation says if it contains config data, then
      # it takes precedence over the rest of the config.
      # If it doesn't, the rest of the PAMDIR config counts.
      result, external = self.EnumerateConfig(None, self.OLD_PAMCONF_FILENAME,
                                              cache)
      if result:
        return result, external

    # If we made it here, there isn't a old-style pam.conf file worth
    # speaking of, so process everything!
    for path in cache:
      # PAM uses the basename as the 'service' id.
      service = os.path.basename(path)
      r, e = self.EnumerateConfig(service, path, cache)
      result.extend(r)
      external.extend(e)
    return result, external


Instruction:
Can you tell what is the following Python 3 function doing
def EnumerateConfig(self, service, path, cache, filter_type=None):

    result = []
    external = []
    path = self._FixPath(path)

    # Make sure we only look at files under PAMDIR.
    # Check we have the file in our artifact/cache. If not, our artifact
    # didn't give it to us, and that's a problem.
    # Note: This should only ever happen if it was referenced
    # from /etc/pam.conf so we can assume that was the file.
    if path not in cache:
      external.append("%s -> %s", self.OLD_PAMCONF_FILENAME, path)
      return result, external

    for tokens in self.ParseEntries(cache[path]):
      if path == self.OLD_PAMCONF_FILENAME:
        # We are processing the old style PAM conf file. It's a special case.
        # It's format is "service type control module-path module-arguments"
        # i.e. the 'service' is the first arg, the rest is line
        # is like everything else except for that addition.
        try:
          service = tokens[0]  # Grab the service from the start line.
          tokens = tokens[1:]  # Make the rest of the line look like "normal".
        except IndexError:
          continue  # It's a blank line, skip it.

      # Process any inclusions in the line.
      new_path = None
      filter_request = None
      try:
        # If a line starts with @include, then include the entire referenced
        # file.
        # e.g. "@include common-auth"
        if tokens[0] == "@include":
          new_path = tokens[1]
        # If a line's second arg is an include/substack, then filter the
        # referenced file only including entries that match the 'type'
        # requested.
        # e.g. "auth include common-auth-screensaver"
        elif tokens[1] in ["include", "substack"]:
          new_path = tokens[2]
          filter_request = tokens[0]
      except IndexError:
        # It's not a valid include line, so keep processing as normal.
        pass

      # If we found an include file, enumerate that file now, and
      # included it where we are in this config file.
      if new_path:
        # Preemptively check to see if we have a problem where the config
        # is referencing a file outside of the expected/defined artifact.
        # Doing it here allows us to produce a better context for the
        # problem. Hence the slight duplication of code.

        new_path = self._FixPath(new_path)
        if new_path not in cache:
          external.append("%s -> %s" % (path, new_path))
          continue  # Skip to the next line of the file.
        r, e = self.EnumerateConfig(service, new_path, cache, filter_request)
        result.extend(r)
        external.extend(e)
      else:
        # If we have been asked to filter on types, skip over any types
        # we are not interested in.
        if filter_type and tokens[0] != filter_type:
          continue  # We can skip this line.

        # If we got here, then we want to include this line in this service's
        # config.

        # Reform the line and break into the correct fields as best we can.
        # Note: ParseEntries doesn't cope with what we need to do.
        match = self.PAMCONF_RE.match(" ".join(tokens))
        if match:
          p_type, control, module_path, module_args = match.group(1, 2, 3, 4)
          # Trim a leading "-" from the type field if present.
          if p_type.startswith("-"):
            p_type = p_type[1:]
          result.append(
              rdf_config_file.PamConfigEntry(
                  service=service,
                  type=p_type,
                  control=control,
                  module_path=module_path,
                  module_args=module_args))
    return result, external

Response:
Return PamConfigEntries it finds as it recursively follows PAM configs.

    Args:
      service: A string containing the service name we are processing.
      path: A string containing the file path name we want.
      cache: A dictionary keyed on path, with the file contents (list of str).
      filter_type: A string containing type name of the results we want.

    Returns:
      A tuple of a list of RDFValue PamConfigEntries found & a list of strings
      which are the external config references found.


Instruction:
How would you implement a function in Python 3 that
validates an AFF4 type.

Response:
def _ValidateAFF4Type(aff4_type):
  """Validates an AFF4 type."""
  if aff4_type is None:
    return

  if not isinstance(aff4_type, type):
    raise TypeError("aff4_type=%s must be a type" % aff4_type)
  if not issubclass(aff4_type, AFF4Object):
    raise TypeError("aff4_type=%s must be a subclass of AFF4Object." %
                    aff4_type)


Instruction:
Given the following Python 3 function, write the documentation
def Open(self, urn, aff4_type=None, mode="r"):
    key = self._ObjectKey(urn, mode)

    try:
      obj = self._objects_cache[key]
    except KeyError:
      obj = FACTORY.Open(
          urn, mode=mode, follow_symlinks=False, token=self._token)
      self._objects_cache[key] = obj

    if aff4_type is not None and not isinstance(obj, aff4_type):
      raise InstantiationError(
          "Object %s is of type %s, but required_type is %s" %
          (urn, obj.__class__.__name__, aff4_type.__name__))

    return obj

Response: Opens the named object in the pool.


Instruction:
Create a Python 3 function for
opening many urns efficiently returning cached objects when possible.

Response:
def MultiOpen(self, urns, aff4_type=None, mode="r"):
    """Opens many urns efficiently, returning cached objects when possible."""
    not_opened_urns = []
    _ValidateAFF4Type(aff4_type)

    for urn in urns:
      key = self._ObjectKey(urn, mode)
      try:
        result = self._objects_cache[key]
        if aff4_type is not None and not isinstance(result, aff4_type):
          continue
        yield result

      except KeyError:
        not_opened_urns.append(urn)

    if not_opened_urns:
      for obj in FACTORY.MultiOpen(
          not_opened_urns, follow_symlinks=False, mode=mode, token=self._token):
        key = self._ObjectKey(obj.urn, mode)
        self._objects_cache[key] = obj

        if aff4_type is not None and not isinstance(obj, aff4_type):
          continue

        yield obj


Instruction:
Can you tell what is the following Python 3 function doing
def ListChildren(self, urn):
    result = self.MultiListChildren([urn])
    try:
      return result[urn]
    except KeyError:
      return []

Response: Lists children of a given urn.


Instruction:
Explain what the following Python 3 code does
def MultiListChildren(self, urns):
    result = {}
    not_listed_urns = []

    for urn in urns:
      try:
        result[urn] = self._children_lists_cache[urn]
      except KeyError:
        not_listed_urns.append(urn)

    if not_listed_urns:
      for urn, children in FACTORY.MultiListChildren(not_listed_urns):
        result[urn] = self._children_lists_cache[urn] = children

      for urn in not_listed_urns:
        self._children_lists_cache.setdefault(urn, [])
        result.setdefault(urn, [])

    return result

Response: Lists children of a bunch of given urns. Results are cached.


Instruction:
Can you tell what is the following Python 3 function doing
def RecursiveMultiListChildren(self, urns):
    result = {}

    checked_urns = set()
    not_cached_urns = []
    urns_to_check = urns
    while True:
      found_children = []

      for urn in urns_to_check:
        try:
          children = result[urn] = self._children_lists_cache[urn]
          found_children.extend(children)
        except KeyError:
          not_cached_urns.append(urn)

      checked_urns.update(urns_to_check)
      urns_to_check = set(found_children) - checked_urns

      if not urns_to_check:
        break

    for urn, children in FACTORY.RecursiveMultiListChildren(not_cached_urns):
      result[urn] = self._children_lists_cache[urn] = children

    return result

Response: Recursively lists given urns. Results are cached.


Instruction:
Given the following Python 3 function, write the documentation
def MultiMarkForDeletion(self, urns):
    all_children_urns = self.RecursiveMultiListChildren(urns)

    urns.extend(collection.Flatten(itervalues(all_children_urns)))
    self._urns_for_deletion.update(urns)

    for obj in self.MultiOpen(urns):
      obj.OnDelete(deletion_pool=self)

Response: Marks multiple urns for deletion.


Instruction:
Can you tell what is the following Python 3 function doing
def root_urns_for_deletion(self):
    roots = set()
    for urn in self._urns_for_deletion:
      new_root = True

      str_urn = utils.SmartUnicode(urn)
      fake_roots = []
      for root in roots:
        str_root = utils.SmartUnicode(root)

        if str_urn.startswith(str_root):
          new_root = False
          break
        elif str_root.startswith(str_urn):
          fake_roots.append(root)

      if new_root:
        roots -= set(fake_roots)
        roots.add(urn)

    return roots

Response: Returns a set of all the root urns marked for deletion.


Instruction:
Can you implement a function in Python 3 that
parses an aff4 age and returns a datastore age specification.

Response:
def ParseAgeSpecification(cls, age):
    """Parses an aff4 age and returns a datastore age specification."""
    try:
      return (0, int(age))
    except (ValueError, TypeError):
      pass

    if age == NEWEST_TIME:
      return data_store.DB.NEWEST_TIMESTAMP
    elif age == ALL_TIMES:
      return data_store.DB.ALL_TIMESTAMPS
    elif len(age) == 2:
      start, end = age

      return (int(start), int(end))

    raise ValueError("Unknown age specification: %s" % age)


Instruction:
Can you tell what is the following Python 3 function doing
def GetAttributes(self, urns, age=NEWEST_TIME):
    urns = set([utils.SmartUnicode(u) for u in urns])
    to_read = {urn: self._MakeCacheInvariant(urn, age) for urn in urns}

    # Urns not present in the cache we need to get from the database.
    if to_read:
      for subject, values in data_store.DB.MultiResolvePrefix(
          to_read,
          AFF4_PREFIXES,
          timestamp=self.ParseAgeSpecification(age),
          limit=None):

        # Ensure the values are sorted.
        values.sort(key=lambda x: x[-1], reverse=True)

        yield utils.SmartUnicode(subject), values

Response: Retrieves all the attributes for all the urns.


Instruction:
Write a Python 3 function for
setting the attributes in the data store.

Response:
def SetAttributes(self,
                    urn,
                    attributes,
                    to_delete,
                    add_child_index=True,
                    mutation_pool=None):
    """Sets the attributes in the data store."""

    attributes[AFF4Object.SchemaCls.LAST] = [
        rdfvalue.RDFDatetime.Now().SerializeToDataStore()
    ]
    to_delete.add(AFF4Object.SchemaCls.LAST)
    if mutation_pool:
      pool = mutation_pool
    else:
      pool = data_store.DB.GetMutationPool()

    pool.MultiSet(urn, attributes, replace=False, to_delete=to_delete)

    if add_child_index:
      self._UpdateChildIndex(urn, pool)
    if mutation_pool is None:
      pool.Flush()


Instruction:
Write a Python 3 script for
updating the child indexes.

Response:
def _UpdateChildIndex(self, urn, mutation_pool):
    """Update the child indexes.

    This function maintains the index for direct child relations. When we set
    an AFF4 path, we always add an attribute like
    index:dir/%(childname)s to its parent. This is written
    asynchronously to its parent.

    In order to query for all direct children of an AFF4 object, we then simple
    get the attributes which match the regex index:dir/.+ which are the
    direct children.

    Args:
      urn: The AFF4 object for which we update the index.
      mutation_pool: A MutationPool object to write to.
    """
    try:
      # Create navigation aids by touching intermediate subject names.
      while urn.Path() != "/":
        basename = urn.Basename()
        dirname = rdfvalue.RDFURN(urn.Dirname())

        try:
          self.intermediate_cache.Get(urn)
          return
        except KeyError:
          extra_attributes = None
          # This is a performance optimization. On the root there is no point
          # setting the last access time since it gets accessed all the time.
          # TODO(amoser): Can we get rid of the index in the root node entirely?
          # It's too big to query anyways...
          if dirname != u"/":
            extra_attributes = {
                AFF4Object.SchemaCls.LAST: [
                    rdfvalue.RDFDatetime.Now().SerializeToDataStore()
                ]
            }

          mutation_pool.AFF4AddChild(
              dirname, basename, extra_attributes=extra_attributes)

          self.intermediate_cache.Put(urn, 1)

          urn = dirname

    except access_control.UnauthorizedAccess:
      pass


Instruction:
Write a Python 3 script for
returning an invariant key for an AFF4 object.

Response:
def _MakeCacheInvariant(self, urn, age):
    """Returns an invariant key for an AFF4 object.

    The object will be cached based on this key. This function is specifically
    extracted to ensure that we encapsulate all security critical aspects of the
    AFF4 object so that objects do not leak across security boundaries.

    Args:
       urn: The urn of the object.
       age: The age policy used to build this object. Should be one of
         ALL_TIMES, NEWEST_TIME or a range.

    Returns:
       A key into the cache.
    """
    precondition.AssertType(urn, Text)
    return "%s:%s" % (urn, self.ParseAgeSpecification(age))


Instruction:
Write a Python 3 function that can
create a new object and locks it.

Response:
def CreateWithLock(self,
                     urn,
                     aff4_type,
                     token=None,
                     age=NEWEST_TIME,
                     force_new_version=True,
                     blocking=True,
                     blocking_lock_timeout=10,
                     blocking_sleep_interval=1,
                     lease_time=100):
    """Creates a new object and locks it.

    Similar to OpenWithLock below, this creates a locked object. The difference
    is that when you call CreateWithLock, the object does not yet have to exist
    in the data store.

    Args:
      urn: The object to create.
      aff4_type: The desired type for this object.
      token: The Security Token to use for opening this item.
      age: The age policy used to build this object. Only makes sense when mode
        has "r".
      force_new_version: Forces the creation of a new object in the data_store.
      blocking: When True, wait and repeatedly try to grab the lock.
      blocking_lock_timeout: Maximum wait time when sync is True.
      blocking_sleep_interval: Sleep time between lock grabbing attempts. Used
        when blocking is True.
      lease_time: Maximum time the object stays locked. Lock will be considered
        released when this time expires.

    Returns:
      An AFF4 object of the desired type and mode.

    Raises:
      AttributeError: If the mode is invalid.
    """

    if not data_store.AFF4Enabled():
      raise NotImplementedError("AFF4 data store has been disabled.")

    transaction = self._AcquireLock(
        urn,
        blocking=blocking,
        blocking_lock_timeout=blocking_lock_timeout,
        blocking_sleep_interval=blocking_sleep_interval,
        lease_time=lease_time)

    # Since we now own the data store subject, we can simply create the aff4
    # object in the usual way.
    return self.Create(
        urn,
        aff4_type,
        mode="rw",
        token=token,
        age=age,
        force_new_version=force_new_version,
        transaction=transaction)


Instruction:
Can you tell what is the following Python 3 function doing
def OpenWithLock(self,
                   urn,
                   aff4_type=None,
                   token=None,
                   age=NEWEST_TIME,
                   blocking=True,
                   blocking_lock_timeout=10,
                   blocking_sleep_interval=1,
                   lease_time=100):
    """Open given urn and locks it.

    Opens an object and locks it for 'lease_time' seconds. OpenWithLock can
    only be used in 'with ...' statement. The lock is released when code
    execution leaves 'with ...' block.

    The urn is always opened in "rw" mode. Symlinks are not followed in
    OpenWithLock() due to possible race conditions.

    Args:
      urn: The urn to open.
      aff4_type: If this optional parameter is set, we raise an
        InstantiationError if the object exists and is not an instance of this
        type. This check is important when a different object can be stored in
        this location.
      token: The Security Token to use for opening this item.
      age: The age policy used to build this object. Should be one of
        NEWEST_TIME, ALL_TIMES or a time range given as a tuple (start, end) in
        microseconds since Jan 1st, 1970.
      blocking: When True, wait and repeatedly try to grab the lock.
      blocking_lock_timeout: Maximum wait time when sync is True.
      blocking_sleep_interval: Sleep time between lock grabbing attempts. Used
        when blocking is True.
      lease_time: Maximum time the object stays locked. Lock will be considered
        released when this time expires.

    Raises:
      ValueError: The URN passed in is None.

    Returns:
      Context manager to be used in 'with ...' statement.
    """

    if not data_store.AFF4Enabled():
      raise NotImplementedError("AFF4 data store has been disabled.")

    transaction = self._AcquireLock(
        urn,
        blocking=blocking,
        blocking_lock_timeout=blocking_lock_timeout,
        blocking_sleep_interval=blocking_sleep_interval,
        lease_time=lease_time)

    # Since we now own the data store subject, we can simply read the aff4
    # object in the usual way.
    return self.Open(
        urn,
        aff4_type=aff4_type,
        mode="rw",
        token=token,
        age=age,
        follow_symlinks=False,
        transaction=transaction)

Response: Opens an object and locks it for lease_time seconds.


Instruction:
Write a Python 3 function for
acquiring a lock for a given URN.

Response:
def _AcquireLock(self,
                   urn,
                   blocking=None,
                   blocking_lock_timeout=None,
                   lease_time=None,
                   blocking_sleep_interval=None):
    """This actually acquires the lock for a given URN."""
    if urn is None:
      raise ValueError("URN cannot be None")

    urn = rdfvalue.RDFURN(urn)

    try:
      return data_store.DB.LockRetryWrapper(
          urn,
          retrywrap_timeout=blocking_sleep_interval,
          retrywrap_max_timeout=blocking_lock_timeout,
          blocking=blocking,
          lease_time=lease_time)
    except data_store.DBSubjectLockError as e:
      raise LockError(e)


Instruction:
How would you implement a function in Python 3 that
makes a copy of one AFF4 object to a different URN.

Response:
def Copy(self,
           old_urn,
           new_urn,
           age=NEWEST_TIME,
           limit=None,
           update_timestamps=False):
    """Make a copy of one AFF4 object to a different URN."""
    new_urn = rdfvalue.RDFURN(new_urn)

    if update_timestamps and age != NEWEST_TIME:
      raise ValueError(
          "Can't update timestamps unless reading the latest version.")

    values = {}
    for predicate, value, ts in data_store.DB.ResolvePrefix(
        old_urn,
        AFF4_PREFIXES,
        timestamp=self.ParseAgeSpecification(age),
        limit=limit):
      if update_timestamps:
        values.setdefault(predicate, []).append((value, None))
      else:
        values.setdefault(predicate, []).append((value, ts))

    if values:
      with data_store.DB.GetMutationPool() as pool:
        pool.MultiSet(new_urn, values, replace=False)
        self._UpdateChildIndex(new_urn, pool)


Instruction:
Write a Python 3 script for
checking if an object with a given URN and type exists in the datastore.

Response:
def ExistsWithType(self,
                     urn,
                     aff4_type=None,
                     follow_symlinks=True,
                     age=NEWEST_TIME,
                     token=None):
    """Checks if an object with a given URN and type exists in the datastore.

    Args:
      urn: The urn to check.
      aff4_type: Expected object type.
      follow_symlinks: If object opened is a symlink, follow it.
      age: The age policy used to check this object. Should be either
        NEWEST_TIME or a time range given as a tuple (start, end) in
        microseconds since Jan 1st, 1970.
      token: The Security Token to use for opening this item.

    Raises:
      ValueError: if aff4_type is not specified.

    Returns:
      True if there's an object with a matching type at a given URN, False
      otherwise.
    """
    if not aff4_type:
      raise ValueError("aff4_type can't be None")

    try:
      self.Open(
          urn,
          aff4_type=aff4_type,
          follow_symlinks=follow_symlinks,
          age=age,
          token=token)
      return True
    except InstantiationError:
      return False


Instruction:
Here you have a function in Python 3, explain what it does
def Open(self,
           urn,
           aff4_type=None,
           mode="r",
           token=None,
           local_cache=None,
           age=NEWEST_TIME,
           follow_symlinks=True,
           transaction=None):
    """Opens the named object.

    This instantiates the object from the AFF4 data store.
    Note that the root aff4:/ object is a container for all other
    objects. Opening it for reading will instantiate a AFF4Volume instance, even
    if the row does not exist.

    The mode parameter specifies, how the object should be opened. A read only
    mode will raise when calling Set() on it, while a write only object will
    never read from the data store. Note that its impossible to open an object
    with pure write support (since we have no idea what type it should be
    without reading the data base) - use Create() instead for purely write mode.

    Args:
      urn: The urn to open.
      aff4_type: If this parameter is set, we raise an IOError if the object is
        not an instance of this type. This check is important when a different
        object can be stored in this location. If mode is "w", this parameter
        will determine the type of the object and is mandatory.
      mode: The mode to open the file with.
      token: The Security Token to use for opening this item.
      local_cache: A dict containing a cache as returned by GetAttributes. If
        set, this bypasses the factory cache.
      age: The age policy used to build this object. Should be one of
        NEWEST_TIME, ALL_TIMES or a time range given as a tuple (start, end) in
        microseconds since Jan 1st, 1970.
      follow_symlinks: If object opened is a symlink, follow it.
      transaction: A lock in case this object is opened under lock.

    Returns:
      An AFF4Object instance.

    Raises:
      IOError: If the object is not of the required type.
      AttributeError: If the requested mode is incorrect.
    """

    if not data_store.AFF4Enabled():
      raise NotImplementedError("AFF4 data store has been disabled.")

    _ValidateAFF4Type(aff4_type)

    if mode not in ["w", "r", "rw"]:
      raise AttributeError("Invalid mode %s" % mode)

    if mode == "w":
      if aff4_type is None:
        raise AttributeError("Need a type to open in write only mode.")
      return self.Create(
          urn,
          aff4_type,
          mode=mode,
          token=token,
          age=age,
          force_new_version=False,
          transaction=transaction)

    urn = rdfvalue.RDFURN(urn)

    if token is None:
      token = data_store.default_token

    if "r" in mode and (local_cache is None or urn not in local_cache):
      local_cache = dict(self.GetAttributes([urn], age=age))

    # Read the row from the table. We know the object already exists if there is
    # some data in the local_cache already for this object.
    result = AFF4Object(
        urn,
        mode=mode,
        token=token,
        local_cache=local_cache,
        age=age,
        follow_symlinks=follow_symlinks,
        object_exists=bool(local_cache.get(urn)),
        transaction=transaction)

    result.aff4_type = aff4_type

    # Now we have a AFF4Object, turn it into the type it is currently supposed
    # to be as specified by Schema.TYPE.
    existing_type = result.Get(result.Schema.TYPE, default="AFF4Volume")

    if existing_type:
      try:
        result = result.Upgrade(AFF4Object.classes[existing_type])
      except KeyError:
        raise InstantiationError("Unable to open %s, type %s unknown." %
                                 (urn, existing_type))

    if aff4_type is not None and not isinstance(result, aff4_type):
      raise InstantiationError(
          "Object %s is of type %s, but required_type is %s" %
          (urn, result.__class__.__name__, aff4_type.__name__))

    return result

Response: Opens the named object in the AFF4 data store.


Instruction:
Can you generate the documentation for the following Python 3 function
def MultiOpen(self,
                urns,
                mode="rw",
                token=None,
                aff4_type=None,
                age=NEWEST_TIME,
                follow_symlinks=True):
    """Opens a bunch of urns efficiently."""

    if not data_store.AFF4Enabled():
      raise NotImplementedError("AFF4 data store has been disabled.")

    if token is None:
      token = data_store.default_token

    if mode not in ["w", "r", "rw"]:
      raise ValueError("Invalid mode %s" % mode)

    symlinks = {}

    _ValidateAFF4Type(aff4_type)

    for urn, values in self.GetAttributes(urns, age=age):
      try:
        obj = self.Open(
            urn,
            mode=mode,
            token=token,
            local_cache={urn: values},
            age=age,
            follow_symlinks=False)
        # We can't pass aff4_type to Open since it will raise on AFF4Symlinks.
        # Setting it here, if needed, so that BadGetAttributeError checking
        # works.
        if aff4_type:
          obj.aff4_type = aff4_type

        if follow_symlinks and isinstance(obj, AFF4Symlink):
          target = obj.Get(obj.Schema.SYMLINK_TARGET)
          if target is not None:
            symlinks.setdefault(target, []).append(obj.urn)
        elif aff4_type:
          if isinstance(obj, aff4_type):
            yield obj
        else:
          yield obj
      except IOError:
        pass

    if symlinks:
      for obj in self.MultiOpen(
          symlinks, mode=mode, token=token, aff4_type=aff4_type, age=age):
        to_link = symlinks[obj.urn]
        for additional_symlink in to_link[1:]:
          clone = obj.__class__(obj.urn, clone=obj)
          clone.symlink_urn = additional_symlink
          yield clone

        obj.symlink_urn = symlinks[obj.urn][0]
        yield obj

Response: Opens a bunch of urns efficiently.


Instruction:
Write a Python 3 function for
opening many URNs and returns handles in the same order.

Response:
def MultiOpenOrdered(self, urns, **kwargs):
    """Opens many URNs and returns handles in the same order.

    `MultiOpen` can return file handles in arbitrary order. This makes it more
    efficient and in most cases the order does not matter. However, there are
    cases where order is important and this function should be used instead.

    Args:
      urns: A list of URNs to open.
      **kwargs: Same keyword arguments as in `MultiOpen`.

    Returns:
      A list of file-like objects corresponding to the specified URNs.

    Raises:
      IOError: If one of the specified URNs does not correspond to the AFF4
          object.
    """
    precondition.AssertIterableType(urns, rdfvalue.RDFURN)

    urn_filedescs = {}
    for filedesc in self.MultiOpen(urns, **kwargs):
      urn_filedescs[filedesc.urn] = filedesc

    filedescs = []

    for urn in urns:
      try:
        filedescs.append(urn_filedescs[urn])
      except KeyError:
        raise IOError("No associated AFF4 object for `%s`" % urn)

    return filedescs


Instruction:
How would you code a function in Python 3 to
return all the versions of the object in the object list.

Response:
def OpenDiscreteVersions(self,
                           urn,
                           mode="r",
                           token=None,
                           local_cache=None,
                           age=ALL_TIMES,
                           diffs_only=False,
                           follow_symlinks=True):
    """Returns all the versions of the object as AFF4 objects.

    This iterates through versions of an object, returning the newest version
    first, then each older version until the beginning of time.

    Note that versions are defined by changes to the TYPE attribute, and this
    takes the version between two TYPE attributes.
    In many cases as a user you don't want this, as you want to be returned an
    object with as many attributes as possible, instead of the subset of them
    that were Set between these two times.

    Args:
      urn: The urn to open.
      mode: The mode to open the file with.
      token: The Security Token to use for opening this item.
      local_cache: A dict containing a cache as returned by GetAttributes. If
        set, this bypasses the factory cache.
      age: The age policy used to build this object. Should be one of ALL_TIMES
        or a time range
      diffs_only: If True, will return only diffs between the versions (instead
        of full objects).
      follow_symlinks: If object opened is a symlink, follow it.

    Yields:
      An AFF4Object for each version.

    Raises:
      IOError: On bad open or wrong time range specified.
    """
    if age == NEWEST_TIME or len(age) == 1:
      raise IOError("Bad age policy NEWEST_TIME for OpenDiscreteVersions.")

    aff4object = FACTORY.Open(
        urn,
        mode=mode,
        token=token,
        local_cache=local_cache,
        age=age,
        follow_symlinks=follow_symlinks)

    # TYPE is always written last so we trust it to bound the version.
    # Iterate from newest to oldest.
    type_iter = aff4object.GetValuesForAttribute(aff4object.Schema.TYPE)
    versions = sorted(t.age for t in type_iter)

    additional_aff4object = None
    # If we want full versions of the object, age != ALL_TIMES and we're
    # dealing with a time range with a start != 0, then we need to fetch
    # the version of the object corresponding to the start of the time
    # range and use its attributes when doing filtering below.
    #
    # Same applies to fetching in DIFF mode and passing a time range in "age".
    # Unversioned arguments will have a timestamp zero and won't be included
    # into the diff. additional_aff4object is then used to account for them.
    if len(age) == 2 and age[0]:
      additional_aff4object = FACTORY.Open(
          urn, mode=mode, token=token, local_cache=local_cache, age=age[0])

    if diffs_only:
      versions.insert(0, rdfvalue.RDFDatetime(0))
      pairs = list(zip(versions, versions[1:]))
    else:
      pairs = [(rdfvalue.RDFDatetime(0), v) for v in versions]

    if pairs:
      right_end = pairs[-1][1]

    for start, end in pairs:
      # Create a subset of attributes for use in the new object that represents
      # this version.
      clone_attrs = {}

      if additional_aff4object:
        # Attributes are supposed to be in the decreasing timestamp
        # order (see aff4.FACTORY.GetAttributes as a reference). Therefore
        # appending additional_aff4object to the end (since it corresponds
        # to the earliest version of the object).
        attributes = itertools.chain(
            iteritems(aff4object.synced_attributes),
            iteritems(additional_aff4object.synced_attributes))
      else:
        attributes = iteritems(aff4object.synced_attributes)

      for k, values in attributes:
        reduced_v = []
        for v in values:
          if v.age > start and v.age <= end:
            reduced_v.append(v)
          elif v.age == 0 and end == right_end:
            v.age = rdfvalue.RDFDatetime(end)
            reduced_v.append(v)
        clone_attrs.setdefault(k, []).extend(reduced_v)

      cur_type = clone_attrs[aff4object.Schema.TYPE][0].ToRDFValue()
      obj_cls = AFF4Object.classes[str(cur_type)]
      new_obj = obj_cls(
          urn,
          mode=mode,
          parent=aff4object.parent,
          clone=clone_attrs,
          token=token,
          age=(start, end),
          local_cache=local_cache,
          follow_symlinks=follow_symlinks)
      new_obj.Initialize()  # This is required to set local attributes.
      yield new_obj


Instruction:
Create a Python 3 function for
returning metadata about all objects in the urns.

Response:
def Stat(self, urns):
    """Returns metadata about all urns.

    Currently the metadata include type, and last update time.

    Args:
      urns: The urns of the objects to open.

    Yields:
      A dict of metadata.

    Raises:
      ValueError: A string was passed instead of an iterable.
    """
    if isinstance(urns, string_types):
      raise ValueError("Expected an iterable, not string.")
    for subject, values in data_store.DB.MultiResolvePrefix(
        urns, ["aff4:type", "metadata:last"]):
      res = dict(urn=rdfvalue.RDFURN(subject))
      for v in values:
        if v[0] == "aff4:type":
          res["type"] = v
        elif v[0] == "metadata:last":
          res["last"] = rdfvalue.RDFDatetime(v[1])
      yield res


Instruction:
Implement a function in Python 3 to
create the object in the data store.

Response:
def Create(self,
             urn,
             aff4_type,
             mode="w",
             token=None,
             age=NEWEST_TIME,
             force_new_version=True,
             object_exists=False,
             mutation_pool=None,
             transaction=None):
    """Creates the urn if it does not already exist, otherwise opens it.

    If the urn exists and is of a different type, this will also promote it to
    the specified type.

    Args:
      urn: The object to create.
      aff4_type: The desired type for this object.
      mode: The desired mode for this object.
      token: The Security Token to use for opening this item.
      age: The age policy used to build this object. Only makes sense when mode
        has "r".
      force_new_version: Forces the creation of a new object in the data_store.
      object_exists: If we know the object already exists we can skip index
        creation.
      mutation_pool: An optional MutationPool object to write to. If not given,
        the data_store is used directly.
      transaction: For locked objects, a lock is passed to the object.

    Returns:
      An AFF4 object of the desired type and mode.

    Raises:
      AttributeError: If the mode is invalid.
    """
    if not data_store.AFF4Enabled():
      raise NotImplementedError("AFF4 data store has been disabled.")

    if mode not in ["w", "r", "rw"]:
      raise AttributeError("Invalid mode %s" % mode)

    if token is None:
      token = data_store.default_token

    if urn is not None:
      urn = rdfvalue.RDFURN(urn)

    _ValidateAFF4Type(aff4_type)

    if "r" in mode:
      # Check to see if an object already exists.
      try:
        existing = self.Open(
            urn, mode=mode, token=token, age=age, transaction=transaction)

        result = existing.Upgrade(aff4_type)

        # We can't pass aff4_type into the Open call since it will raise with a
        # type mismatch. We set it like this so BadGetAttributeError checking
        # works.
        if aff4_type:
          result.aff4_type = aff4_type.__name__

        if force_new_version and existing.Get(
            result.Schema.TYPE) != aff4_type.__name__:
          result.ForceNewVersion()
        return result
      except IOError:
        pass

    result = aff4_type(
        urn,
        mode=mode,
        token=token,
        age=age,
        aff4_type=aff4_type.__name__,
        object_exists=object_exists,
        mutation_pool=mutation_pool,
        transaction=transaction)
    result.Initialize()
    if force_new_version:
      result.ForceNewVersion()

    return result


Instruction:
Write a Python 3 script to
drop all the information about given objects. DANGEROUS! This recursively deletes all objects contained within the specified URN. Args: urns: Urns of objects to remove. token: The Security Token to use for opening this item. Raises: ValueError: If one of the urns is too short. This is a safety check to ensure the root is not removed.

Response:
def MultiDelete(self, urns, token=None):
    """Drop all the information about given objects.

    DANGEROUS! This recursively deletes all objects contained within the
    specified URN.

    Args:
      urns: Urns of objects to remove.
      token: The Security Token to use for opening this item.

    Raises:
      ValueError: If one of the urns is too short. This is a safety check to
      ensure the root is not removed.
    """
    urns = [rdfvalue.RDFURN(urn) for urn in urns]

    if token is None:
      token = data_store.default_token

    for urn in urns:
      if urn.Path() == "/":
        raise ValueError("Can't delete root URN. Please enter a valid URN")

    deletion_pool = DeletionPool(token=token)
    deletion_pool.MultiMarkForDeletion(urns)

    marked_root_urns = deletion_pool.root_urns_for_deletion
    marked_urns = deletion_pool.urns_for_deletion

    logging.debug(u"Found %d objects to remove when removing %s",
                  len(marked_urns), urns)

    logging.debug(u"Removing %d root objects when removing %s: %s",
                  len(marked_root_urns), urns, marked_root_urns)

    pool = data_store.DB.GetMutationPool()
    for root in marked_root_urns:
      # Only the index of the parent object should be updated. Everything
      # below the target object (along with indexes) is going to be
      # deleted.
      self._DeleteChildFromIndex(root, mutation_pool=pool)

    for urn_to_delete in marked_urns:
      try:
        self.intermediate_cache.ExpireObject(urn_to_delete.Path())
      except KeyError:
        pass

    pool.DeleteSubjects(marked_urns)
    pool.Flush()

    # Ensure this is removed from the cache as well.
    self.Flush()

    logging.debug("Removed %d objects", len(marked_urns))


Instruction:
Make a summary of the following Python 3 code
def MultiListChildren(self, urns, limit=None, age=NEWEST_TIME):
    checked_subjects = set()

    for subject, values in data_store.DB.AFF4MultiFetchChildren(
        urns, timestamp=Factory.ParseAgeSpecification(age), limit=limit):

      checked_subjects.add(subject)

      subject_result = []
      for child, timestamp in values:
        urn = rdfvalue.RDFURN(subject).Add(child)
        urn.age = rdfvalue.RDFDatetime(timestamp)
        subject_result.append(urn)

      yield subject, subject_result

    for subject in set(urns) - checked_subjects:
      yield subject, []

Response: Lists bunch of directories efficiently.


Instruction:
Can you tell what is the following Python 3 function doing
def ListChildren(self, urn, limit=None, age=NEWEST_TIME):
    _, children_urns = list(
        self.MultiListChildren([urn], limit=limit, age=age))[0]
    return children_urns

Response: Lists the children of a given resource.


Instruction:
How would you explain what the following Python 3 function does
def RecursiveMultiListChildren(self, urns, limit=None, age=NEWEST_TIME):

    checked_urns = set()
    urns_to_check = urns
    while True:
      found_children = []

      for subject, values in self.MultiListChildren(
          urns_to_check, limit=limit, age=age):

        found_children.extend(values)
        yield subject, values

      checked_urns.update(urns_to_check)

      urns_to_check = set(found_children) - checked_urns
      if not urns_to_check:
        break

Response: Recursively lists a bunch of directories.


Instruction:
Write a Python 3 function for
returning a copy without registering in the attribute registry.

Response:
def Copy(self):
    """Return a copy without registering in the attribute registry."""
    return Attribute(
        self.predicate,
        self.attribute_type,
        self.description,
        self.name,
        _copy=True)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def GetRDFValueType(self):
    result = self.attribute_type
    for field_name in self.field_names:
      # Support the new semantic protobufs.
      try:
        result = result.type_infos.get(field_name).type
      except AttributeError:
        raise AttributeError("Invalid attribute %s" % field_name)

    return result

Response: Returns this attribute s RDFValue class.


Instruction:
Explain what the following Python 3 code does
def GetSubFields(self, fd, field_names):

    if isinstance(fd, rdf_protodict.RDFValueArray):
      for value in fd:
        for res in self._GetSubField(value, field_names):
          yield res
    else:
      for res in self._GetSubField(fd, field_names):
        yield res

Response: Gets all the subfields indicated by field_names.


Instruction:
How would you code a function in Python 3 to
return the values for this attribute as stored in an AFF4Object.

Response:
def GetValues(self, fd):
    """Return the values for this attribute as stored in an AFF4Object."""
    result = None
    for result in fd.new_attributes.get(self, []):
      # We need to interpolate sub fields in this rdfvalue.
      if self.field_names:
        for x in self.GetSubFields(result, self.field_names):
          yield x

      else:
        yield result

    for result in fd.synced_attributes.get(self, []):
      result = result.ToRDFValue()

      # We need to interpolate sub fields in this rdfvalue.
      if result is not None:
        if self.field_names:
          for x in self.GetSubFields(result, self.field_names):
            yield x

        else:
          yield result

    if result is None:
      default = self.GetDefault(fd)
      if default is not None:
        yield default


Instruction:
How would you code a function in Python 3 to
return a default attribute if it is not set.

Response:
def GetDefault(self, fd=None, default=None):
    """Returns a default attribute if it is not set."""
    if callable(self.default):
      return self.default(fd)

    if self.default is not None:
      # We can't return mutable objects here or the default might change for all
      # objects of this class.
      if isinstance(self.default, rdfvalue.RDFValue):
        default = self.default.Copy()
        default.attribute_instance = self
        return self(default)
      else:
        return self(self.default)

    if isinstance(default, rdfvalue.RDFValue):
      default = default.Copy()
      default.attribute_instance = self

    return default


Instruction:
Write a Python 3 function that can
give a serialized value decode the attribute.

Response:
def DecodeValueFromAttribute(self, attribute_name, value, ts):
    """Given a serialized value, decode the attribute.

    Only attributes which have been previously defined are permitted.

    Args:
       attribute_name: The string name of the attribute.
       value: The serialized attribute value.
       ts: The timestamp of this attribute.
    """
    try:
      # Get the Attribute object from our schema.
      attribute = Attribute.PREDICATES[attribute_name]
      cls = attribute.attribute_type
      self._AddAttributeToCache(attribute, LazyDecoder(cls, value, ts),
                                self.synced_attributes)
    except KeyError:
      pass
    except (ValueError, rdfvalue.DecodeError):
      logging.debug("%s: %s invalid encoding. Skipping.", self.urn,
                    attribute_name)


Instruction:
Given the following Python 3 function, write the documentation
def UpdateLease(self, duration):
    if not self.locked:
      raise LockError("Object must be locked to update the lease: %s." %
                      self.urn)

    if self.CheckLease() == 0:
      self._RaiseLockError("UpdateLease")

    self.transaction.UpdateLease(duration)

Response: Updates the lease and flushes the object.


Instruction:
Create a Python 3 function for
closing and destroys the object.

Response:
def Close(self):
    """Close and destroy the object.

    This is similar to Flush, but does not maintain object validity. Hence the
    object should not be interacted with after Close().

    Raises:
       LockError: The lease for this object has expired.
    """
    if self.locked and self.CheckLease() == 0:
      raise LockError("Can not update lease that has already expired.")

    self._WriteAttributes()

    # Releasing this lock allows another thread to own it.
    if self.locked:
      self.transaction.Release()

    if self.parent:
      self.parent.Close()

    # Interacting with a closed object is a bug. We need to catch this ASAP so
    # we remove all mode permissions from this object.
    self.mode = ""


Instruction:
Can you implement a function in Python 3 that
writes the dirty attributes to the data store.

Response:
def _WriteAttributes(self):
    """Write the dirty attributes to the data store."""
    # If the object is not opened for writing we do not need to flush it to the
    # data_store.
    if "w" not in self.mode:
      return

    if self.urn is None:
      raise ValueError("Storing of anonymous AFF4 objects not supported.")

    to_set = {}
    for attribute_name, value_array in iteritems(self.new_attributes):
      to_set_list = to_set.setdefault(attribute_name, [])
      for value in value_array:
        to_set_list.append((value.SerializeToDataStore(), value.age))

    if self._dirty:
      # We determine this object has a new version only if any of the versioned
      # attributes have changed. Non-versioned attributes do not represent a new
      # object version. The type of an object is versioned and represents a
      # version point in the life of the object.
      if self._new_version:
        to_set[self.Schema.TYPE] = [
            (utils.SmartUnicode(self.__class__.__name__),
             rdfvalue.RDFDatetime.Now())
        ]

      # We only update indexes if the schema does not forbid it and we are not
      # sure that the object already exists.
      add_child_index = self.Schema.ADD_CHILD_INDEX
      if self.object_exists:
        add_child_index = False

      # Write the attributes to the Factory cache.
      FACTORY.SetAttributes(
          self.urn,
          to_set,
          self._to_delete,
          add_child_index=add_child_index,
          mutation_pool=self.mutation_pool)


Instruction:
Can you tell what is the following Python 3 function doing
def _SyncAttributes(self):
    # This effectively moves all the values from the new_attributes to the
    # synced_attributes caches.
    for attribute, value_array in iteritems(self.new_attributes):
      if not attribute.versioned or self.age_policy == NEWEST_TIME:
        # Store the latest version if there are multiple unsynced versions.
        value = value_array[-1]
        self.synced_attributes[attribute] = [
            LazyDecoder(decoded=value, age=value.age)
        ]

      else:
        synced_value_array = self.synced_attributes.setdefault(attribute, [])
        for value in value_array:
          synced_value_array.append(LazyDecoder(decoded=value, age=value.age))

        synced_value_array.sort(key=lambda x: x.age, reverse=True)

    self.new_attributes = {}
    self._to_delete.clear()
    self._dirty = False
    self._new_version = False

Response: Sync the new attributes to the synced attribute cache.


Instruction:
Create a Python 3 function to
check that the value for the given attribute is of the expected type.

Response:
def _CheckAttribute(self, attribute, value):
    """Check that the value is of the expected type.

    Args:
       attribute: An instance of Attribute().
       value: An instance of RDFValue.

    Raises:
       ValueError: when the value is not of the expected type.
       AttributeError: When the attribute is not of type Attribute().
    """
    if not isinstance(attribute, Attribute):
      raise AttributeError("Attribute %s must be of type aff4.Attribute()" %
                           attribute)

    if not isinstance(value, attribute.attribute_type):
      raise ValueError("Value for attribute %s must be of type %s()" %
                       (attribute, attribute.attribute_type.__name__))


Instruction:
How would you code a function in Python 3 to
add an additional attribute to this object.

Response:
def AddAttribute(self, attribute, value=None, age=None):
    """Add an additional attribute to this object.

    If value is None, attribute is expected to be already initialized with a
    value. For example:

    fd.AddAttribute(fd.Schema.CONTAINS("some data"))

    Args:
       attribute: The attribute name or an RDFValue derived from the attribute.
       value: The value the attribute will be set to.
       age: Age (timestamp) of the attribute. If None, current time is used.

    Raises:
       IOError: If this object is read only.
    """
    if "w" not in self.mode:
      raise IOError("Writing attribute %s to read only object." % attribute)

    if value is None:
      value = attribute
      attribute = value.attribute_instance

    # Check if this object should be locked in order to add the attribute.
    # NOTE: We don't care about locking when doing blind writes.
    if self.mode != "w" and attribute.lock_protected and not self.transaction:
      raise IOError("Object must be locked to write attribute %s." % attribute)

    self._CheckAttribute(attribute, value)
    # Does this represent a new version?
    if attribute.versioned:
      if attribute.creates_new_object_version:
        self._new_version = True

      # Update the time of this new attribute.
      if age:
        value.age = age
      else:
        value.age = rdfvalue.RDFDatetime.Now()

    # Non-versioned attributes always replace previous versions and get written
    # at the earliest timestamp (so they appear in all objects).
    else:
      self._to_delete.add(attribute)
      self.synced_attributes.pop(attribute, None)
      self.new_attributes.pop(attribute, None)
      value.age = 0

    self._AddAttributeToCache(attribute, value, self.new_attributes)
    self._dirty = True


Instruction:
Create a Python 3 function for
clearing the attribute from this object.

Response:
def DeleteAttribute(self, attribute):
    """Clears the attribute from this object."""
    if "w" not in self.mode:
      raise IOError("Deleting attribute %s from read only object." % attribute)

    # Check if this object should be locked in order to delete the attribute.
    # NOTE: We don't care about locking when doing blind writes.
    if self.mode != "w" and attribute.lock_protected and not self.transaction:
      raise IOError("Object must be locked to delete attribute %s." % attribute)

    if attribute in self.synced_attributes:
      self._to_delete.add(attribute)
      del self.synced_attributes[attribute]

    if attribute in self.new_attributes:
      del self.new_attributes[attribute]

    # Does this represent a new version?
    if attribute.versioned and attribute.creates_new_object_version:
      self._new_version = True

    self._dirty = True


Instruction:
Given the following Python 3 function, write the documentation
def Get(self, attribute, default=None):
    if attribute is None:
      return default

    # Allow the user to specify the attribute by name.
    elif isinstance(attribute, str):
      attribute = Attribute.GetAttributeByName(attribute)

    # We can't read attributes from the data_store unless read mode was
    # specified. It is ok to read new attributes though.
    if "r" not in self.mode and (attribute not in self.new_attributes and
                                 attribute not in self.synced_attributes):
      raise IOError("Fetching %s from object not opened for reading." %
                    attribute)

    for result in self.GetValuesForAttribute(attribute, only_one=True):
      try:
        # The attribute may be a naked string or int - i.e. not an RDFValue at
        # all.
        result.attribute_instance = attribute
      except AttributeError:
        pass

      return result

    return attribute.GetDefault(self, default)

Response: Gets the attribute from this object.


Instruction:
Can you write a function in Python 3 where it
returns a list of values from this attribute.

Response:
def GetValuesForAttribute(self, attribute, only_one=False):
    """Returns a list of values from this attribute."""
    if not only_one and self.age_policy == NEWEST_TIME:
      raise ValueError("Attempting to read all attribute versions for an "
                       "object opened for NEWEST_TIME. This is probably "
                       "not what you want.")

    if attribute is None:
      return []

    elif isinstance(attribute, string_types):
      attribute = Attribute.GetAttributeByName(attribute)

    return attribute.GetValues(self)


Instruction:
Here you have a function in Python 3, explain what it does
def Upgrade(self, aff4_class):
    _ValidateAFF4Type(aff4_class)

    # We are already of the required type
    if self.__class__ == aff4_class:
      return self

    # Check that we have the right type.
    if not isinstance(aff4_class, type):
      raise InstantiationError("aff4_class=%s must be a type" % aff4_class)
    if not issubclass(aff4_class, AFF4Object):
      raise InstantiationError(
          "aff4_class=%s must be a subclass of AFF4Object." % aff4_class)

    # It's not allowed to downgrade the object
    if isinstance(self, aff4_class):
      # TODO(user): check what we should do here:
      #                 1) Nothing
      #                 2) raise
      #                 3) return self
      # Option 3) seems ok, but we need to be sure that we don't use
      # Create(mode='r') anywhere where code actually expects the object to be
      # downgraded.
      return self

    # NOTE: It is possible for attributes to become inaccessible here if the old
    # object has an attribute which the new object does not have in its
    # schema. The values of these attributes will not be available any longer in
    # the new object - usually because old attributes do not make sense in the
    # context of the new object.

    # Instantiate the class
    result = aff4_class(
        self.urn,
        mode=self.mode,
        clone=self,
        parent=self.parent,
        token=self.token,
        age=self.age_policy,
        object_exists=self.object_exists,
        follow_symlinks=self.follow_symlinks,
        aff4_type=self.aff4_type,
        mutation_pool=self.mutation_pool,
        transaction=self.transaction)
    result.symlink_urn = self.urn
    result.Initialize()

    return result

Response: Upgrades this object to the type specified.


Instruction:
Here you have a function in Python 3, explain what it does
def AddLabels(self, labels_names, owner=None):
    if owner is None and not self.token:
      raise ValueError("Can't set label: No owner specified and "
                       "no access token available.")
    if isinstance(labels_names, string_types):
      raise ValueError("Label list can't be string.")

    owner = owner or self.token.username

    current_labels = self.Get(self.Schema.LABELS, self.Schema.LABELS())
    for label_name in labels_names:
      label = rdf_aff4.AFF4ObjectLabel(
          name=label_name, owner=owner, timestamp=rdfvalue.RDFDatetime.Now())
      current_labels.AddLabel(label)

    self.Set(current_labels)

Response: Adds labels to the AFF4Object.


Instruction:
Implement a Python 3 function for
removing specified labels from the AFF4Object.

Response:
def RemoveLabels(self, labels_names, owner=None):
    """Remove specified labels from the AFF4Object."""
    if owner is None and not self.token:
      raise ValueError("Can't remove label: No owner specified and "
                       "no access token available.")
    if isinstance(labels_names, string_types):
      raise ValueError("Label list can't be string.")

    owner = owner or self.token.username

    current_labels = self.Get(self.Schema.LABELS)
    for label_name in labels_names:
      label = rdf_aff4.AFF4ObjectLabel(name=label_name, owner=owner)
      current_labels.RemoveLabel(label)

    self.Set(self.Schema.LABELS, current_labels)


Instruction:
Write a Python 3 function for
checking that attribute is a valid Attribute instance.

Response:
def SetAttribute(self, attribute):
    """Checks that attribute is a valid Attribute() instance."""
    # Grab the attribute registered for this name
    self.attribute = attribute
    self.attribute_obj = Attribute.GetAttributeByName(attribute)
    if self.attribute_obj is None:
      raise lexer.ParseError("Attribute %s not defined" % attribute)


Instruction:
Here you have a function in Python 3, explain what it does
def SetOperator(self, operator):
    self.operator = operator
    # Find the appropriate list of operators for this attribute
    attribute_type = self.attribute_obj.GetRDFValueType()
    operators = attribute_type.operators

    # Do we have such an operator?
    self.number_of_args, self.operator_method = operators.get(
        operator, (0, None))

    if self.operator_method is None:
      raise lexer.ParseError("Operator %s not defined on attribute '%s'" %
                             (operator, self.attribute))

    self.operator_method = getattr(attribute_type, self.operator_method)

Response: Sets the operator for this expression.


Instruction:
Write a Python 3 script to
return the data_store filter implementation from the attribute.

Response:
def Compile(self, filter_implemention):
    """Returns the data_store filter implementation from the attribute."""
    return self.operator_method(self.attribute_obj, filter_implemention,
                                *self.args)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def ListChildren(self, limit=None, age=NEWEST_TIME):
    # Just grab all the children from the index.
    for predicate, timestamp in data_store.DB.AFF4FetchChildren(
        self.urn, timestamp=Factory.ParseAgeSpecification(age), limit=limit):
      urn = self.urn.Add(predicate)
      urn.age = rdfvalue.RDFDatetime(timestamp)
      yield urn

Response: Returns an iterator over the children of this object.


Instruction:
Implement a function in Python 3 to
return an AFF4 object for all our direct children.

Response:
def OpenChildren(self,
                   children=None,
                   mode="r",
                   limit=None,
                   chunk_limit=100000,
                   age=NEWEST_TIME):
    """Yields AFF4 Objects of all our direct children.

    This method efficiently returns all attributes for our children directly, in
    a few data store round trips. We use the directory indexes to query the data
    store.

    Args:
      children: A list of children RDFURNs to open. If None open all our
        children.
      mode: The mode the files should be opened with.
      limit: Total number of items we will attempt to retrieve.
      chunk_limit: Maximum number of items to retrieve at a time.
      age: The age of the items to retrieve. Should be one of ALL_TIMES,
        NEWEST_TIME or a range.

    Yields:
      Instances for each direct child.
    """
    if children is None:
      # No age passed here to avoid ignoring indexes that were updated
      # to a timestamp greater than the object's age.
      subjects = list(self.ListChildren())
    else:
      subjects = list(children)
    subjects.sort()
    result_count = 0
    # Read at most limit children at a time.
    while subjects:
      to_read = subjects[:chunk_limit]
      subjects = subjects[chunk_limit:]
      for child in FACTORY.MultiOpen(
          to_read, mode=mode, token=self.token, age=age):
        yield child
        result_count += 1
        if limit and result_count >= limit:
          return


Instruction:
Write a Python 3 script to
return a pathspec for an aff4 object even if there is none stored.

Response:
def real_pathspec(self):
    """Returns a pathspec for an aff4 object even if there is none stored."""
    pathspec = self.Get(self.Schema.PATHSPEC)

    stripped_components = []
    parent = self

    # TODO(user): this code is potentially slow due to multiple separate
    # aff4.FACTORY.Open() calls. OTOH the loop below is executed very rarely -
    # only when we deal with deep files that got fetched alone and then
    # one of the directories in their path gets updated.
    while not pathspec and len(parent.urn.Split()) > 1:
      # We try to recurse up the tree to get a real pathspec.
      # These directories are created automatically without pathspecs when a
      # deep directory is listed without listing the parents.
      # Note /fs/os or /fs/tsk won't be updateable so we will raise IOError
      # if we try.
      stripped_components.append(parent.urn.Basename())
      pathspec = parent.Get(parent.Schema.PATHSPEC)
      parent = FACTORY.Open(parent.urn.Dirname(), token=self.token)

    if pathspec:
      if stripped_components:
        # We stripped pieces of the URL, time to add them back.
        new_path = utils.JoinPath(*reversed(stripped_components[:-1]))
        pathspec.Append(
            rdf_paths.PathSpec(path=new_path, pathtype=pathspec.last.pathtype))
    else:
      raise IOError("Item has no pathspec.")

    return pathspec


Instruction:
Explain what the following Python 3 code does
def _MultiStream(cls, fds):
    for fd in fds:
      fd.Seek(0)
      while True:
        chunk = fd.Read(cls.MULTI_STREAM_CHUNK_SIZE)
        if not chunk:
          break
        yield fd, chunk, None

Response: Method overriden by subclasses to optimize the MultiStream behavior.


Instruction:
Given the following Python 3 function, write the documentation
def MultiStream(cls, fds):
    for fd in fds:
      if not isinstance(fd, AFF4Stream):
        raise ValueError("All object to be streamed have to inherit from "
                         "AFF4Stream (found one inheriting from %s)." %
                         (fd.__class__.__name__))

    classes_map = {}
    for fd in fds:
      classes_map.setdefault(fd.__class__, []).append(fd)

    for fd_class, fds in iteritems(classes_map):
      # pylint: disable=protected-access
      for fd, chunk, exception in fd_class._MultiStream(fds):
        yield fd, chunk, exception

Response: Effectively streams data from multiple opened AFF4Streams objects.


Instruction:
How would you explain what the following Python 3 function does
def Initialize(self):
    super(AFF4MemoryStreamBase, self).Initialize()
    contents = b""

    if "r" in self.mode:
      contents = self.Get(self.Schema.CONTENT).AsBytes()
      try:
        if contents is not None:
          contents = zlib.decompress(contents)
      except zlib.error:
        pass

    self.fd = io.BytesIO(contents)
    self.size = len(contents)
    self.offset = 0

Response: Try to load the data from the store.


Instruction:
Here you have a function in Python 3, explain what it does
def _MultiStream(cls, fds):

    missing_chunks_by_fd = {}
    for chunk_fd_pairs in collection.Batch(
        cls._GenerateChunkPaths(fds), cls.MULTI_STREAM_CHUNKS_READ_AHEAD):

      chunks_map = dict(chunk_fd_pairs)
      contents_map = {}
      for chunk_fd in FACTORY.MultiOpen(
          chunks_map, mode="r", token=fds[0].token):
        if isinstance(chunk_fd, AFF4Stream):
          fd = chunks_map[chunk_fd.urn]
          contents_map[chunk_fd.urn] = chunk_fd.read()

      for chunk_urn, fd in chunk_fd_pairs:
        if chunk_urn not in contents_map or not contents_map[chunk_urn]:
          missing_chunks_by_fd.setdefault(fd, []).append(chunk_urn)

      for chunk_urn, fd in chunk_fd_pairs:
        if fd in missing_chunks_by_fd:
          continue

        yield fd, contents_map[chunk_urn], None

    for fd, missing_chunks in iteritems(missing_chunks_by_fd):
      e = MissingChunksError(
          "%d missing chunks (multi-stream)." % len(missing_chunks),
          missing_chunks=missing_chunks)
      yield fd, None, e

Response: Effectively streams data from multiple opened AFF4ImageBase objects.


Instruction:
How would you explain what the following Python 3 function does
def Initialize(self):
    super(AFF4ImageBase, self).Initialize()
    self.offset = 0
    # A cache for segments.
    self.chunk_cache = ChunkCache(self._WriteChunk, 100)

    if "r" in self.mode:
      self.size = int(self.Get(self.Schema.SIZE))
      # pylint: disable=protected-access
      self.chunksize = int(self.Get(self.Schema._CHUNKSIZE))
      # pylint: enable=protected-access
      self.content_last = self.Get(self.Schema.CONTENT_LAST)
    else:
      self.size = 0
      self.content_last = None

Response: Initialize the AFF4ImageBase object.


Instruction:
Write a Python 3 script for
reading as much as possible but not more than length.

Response:
def _ReadPartial(self, length):
    """Read as much as possible, but not more than length."""
    chunk = self.offset // self.chunksize
    chunk_offset = self.offset % self.chunksize

    available_to_read = min(length, self.chunksize - chunk_offset)

    retries = 0
    while retries < self.NUM_RETRIES:
      fd = self._GetChunkForReading(chunk)
      if fd:
        break
      # Arriving here means we know about blobs that cannot be found in the db.
      # The most likely reason is that they have not been synced yet so we
      # retry a couple of times just in case they come in eventually.
      logging.warning("Chunk not found.")
      time.sleep(1)
      retries += 1

    if retries >= self.NUM_RETRIES:
      raise IOError("Chunk not found for reading.")

    fd.seek(chunk_offset)

    result = fd.read(available_to_read)
    self.offset += len(result)

    return result


Instruction:
Explain what the following Python 3 code does
def Read(self, length):
    result = b""

    # The total available size in the file
    length = int(length)
    length = min(length, self.size - self.offset)

    while length > 0:
      data = self._ReadPartial(length)
      if not data:
        break

      length -= len(data)
      result += data
    return result

Response: Reads a block of data from the file.


Instruction:
How would you explain what the following Python 3 function does
def _WritePartial(self, data):

    chunk = self.offset // self.chunksize
    chunk_offset = self.offset % self.chunksize
    data = utils.SmartStr(data)

    available_to_write = min(len(data), self.chunksize - chunk_offset)

    fd = self._GetChunkForWriting(chunk)
    fd.seek(chunk_offset)

    fd.write(data[:available_to_write])
    self.offset += available_to_write

    return data[available_to_write:]

Response: Writes at most one chunk of data.


Instruction:
Can you generate the documentation for the following Python 3 function
def Encode(self, attribute, value):
    required_type = self._attribute_types.get(attribute, "bytes")

    if required_type == "integer":
      return rdf_structs.SignedVarintEncode(int(value))
    elif required_type == "unsigned_integer":
      return rdf_structs.VarintEncode(int(value))
    elif hasattr(value, "SerializeToString"):
      return value.SerializeToString()
    elif isinstance(value, bytes):
      return value
    elif isinstance(value, Text):
      return value.encode("utf-8")
    else:
      return str(value).encode("utf-8")

Response: Encodes the value for the attribute.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def Decode(self, attribute, value):
    required_type = self._attribute_types.get(attribute, "bytes")
    if required_type == "integer":
      return rdf_structs.SignedVarintReader(value, 0)[0]
    elif required_type == "unsigned_integer":
      return rdf_structs.VarintReader(value, 0)[0]
    elif required_type == "string":
      if isinstance(value, bytes):
        return value.decode("utf-8")
      else:
        return utils.SmartUnicode(value)
    else:
      return value

Response: Decode the value to the required type.


Instruction:
Here you have a function in Python 3, explain what it does
def CleanAff4Hunts(self):

    hunts_ttl = config.CONFIG["DataRetention.hunts_ttl"]
    if not hunts_ttl:
      self.Log("TTL not set - nothing to do...")
      return

    exception_label = config.CONFIG["DataRetention.hunts_ttl_exception_label"]

    hunts_root = aff4.FACTORY.Open("aff4:/hunts", token=self.token)
    hunts_urns = list(hunts_root.ListChildren())

    deadline = rdfvalue.RDFDatetime.Now() - hunts_ttl

    hunts_deleted = 0

    hunts = aff4.FACTORY.MultiOpen(
        hunts_urns, aff4_type=implementation.GRRHunt, token=self.token)
    for hunt in hunts:
      if exception_label in hunt.GetLabelsNames():
        continue

      runner = hunt.GetRunner()
      if runner.context.create_time + runner.context.duration < deadline:
        aff4.FACTORY.Delete(hunt.urn, token=self.token)
        hunts_deleted += 1
        self.HeartBeat()
    self.Log("Deleted %d hunts." % hunts_deleted)

Response: Cleans up old hunt data from aff4.


Instruction:
Given the following Python 3 function, write the documentation
def Start(self):
    cron_jobs_ttl = config.CONFIG["DataRetention.cron_jobs_flows_ttl"]
    if not cron_jobs_ttl:
      self.Log("TTL not set - nothing to do...")
      return

    manager = aff4_cronjobs.GetCronManager()
    cutoff_timestamp = rdfvalue.RDFDatetime.Now() - cron_jobs_ttl
    if data_store.RelationalDBEnabled():
      deletion_count = manager.DeleteOldRuns(cutoff_timestamp=cutoff_timestamp)

    else:
      deletion_count = 0
      for job in manager.ReadJobs(token=self.token):
        deletion_count += manager.DeleteOldRuns(
            job, cutoff_timestamp=cutoff_timestamp, token=self.token)
        self.HeartBeat()

    self.Log("Deleted %d cron job runs." % deletion_count)

Response: Cleans up old cron job data.


Instruction:
How would you explain what the following Python 3 function does
def CleanAff4Clients(self):

    inactive_client_ttl = config.CONFIG["DataRetention.inactive_client_ttl"]
    if not inactive_client_ttl:
      self.Log("TTL not set - nothing to do...")
      return

    exception_label = config.CONFIG[
        "DataRetention.inactive_client_ttl_exception_label"]

    index = client_index.CreateClientIndex(token=self.token)

    client_urns = index.LookupClients(["."])

    deadline = rdfvalue.RDFDatetime.Now() - inactive_client_ttl
    deletion_count = 0

    for client_group in collection.Batch(client_urns, 1000):
      inactive_client_urns = []
      for client in aff4.FACTORY.MultiOpen(
          client_group,
          mode="r",
          aff4_type=aff4_grr.VFSGRRClient,
          token=self.token):
        if exception_label in client.GetLabelsNames():
          continue

        if client.Get(client.Schema.LAST) < deadline:
          inactive_client_urns.append(client.urn)

      aff4.FACTORY.MultiDelete(inactive_client_urns, token=self.token)
      deletion_count += len(inactive_client_urns)
      self.HeartBeat()

    self.Log("Deleted %d inactive clients." % deletion_count)

Response: Cleans up old client data from aff4.


Instruction:
Can you implement a function in Python 3 that
does basic token validation. Args: token: User's credentials as access_control.ACLToken. targets: List of targets that were meant to be accessed by the token. This is used for logging purposes only. Returns: True if token is valid. Raises: access_control.UnauthorizedAccess: if token is not valid. ValueError: if targets list is empty.

Response:
def ValidateToken(token, targets):
  """Does basic token validation.

  Args:
    token: User's credentials as access_control.ACLToken.
    targets: List of targets that were meant to be accessed by the token. This
      is used for logging purposes only.

  Returns:
    True if token is valid.

  Raises:
    access_control.UnauthorizedAccess: if token is not valid.
    ValueError: if targets list is empty.
  """

  def GetSubjectForError():
    if len(targets) == 1:
      return list(targets)[0]
    else:
      return None

  # All accesses need a token.
  if not token:
    raise access_control.UnauthorizedAccess(
        "Must give an authorization token for %s" % targets,
        subject=GetSubjectForError())

  # Token must not be expired here.
  token.CheckExpiry()

  # Token must have identity
  if not token.username:
    raise access_control.UnauthorizedAccess(
        "Must specify a username for access to %s." % targets,
        subject=GetSubjectForError())

  return True


Instruction:
How would you implement a function in Python 3 that
validates that the requested access and subjects are valid.

Response:
def ValidateAccessAndSubjects(requested_access, subjects):
  """Does basic requested access validation.

  Args:
    requested_access: String consisting or 'r', 'w' and 'q' characters.
    subjects: A list of subjects that are about to be accessed with a given
      requested_access. Used for logging purposes only.

  Returns:
    True if requested_access is valid.

  Raises:
    access_control.UnauthorizedAccess: if requested_access is not valid.
    ValueError: if subjects list is empty.
  """

  if not requested_access:
    raise access_control.UnauthorizedAccess(
        "Must specify requested access type for %s" % subjects)

  for s in requested_access:
    if s not in "rwq":
      raise ValueError(
          "Invalid access requested for %s: %s" % (subjects, requested_access))

  if "q" in requested_access and "r" not in requested_access:
    raise access_control.UnauthorizedAccess(
        "Invalid access request: query permissions require read permissions "
        "for %s" % subjects,
        requested_access=requested_access)

  return True


Instruction:
Explain what the following Python 3 code does
def CheckUserForLabels(username, authorized_labels, token=None):
  authorized_labels = set(authorized_labels)

  try:
    user = aff4.FACTORY.Open(
        "aff4:/users/%s" % username, aff4_type=aff4_users.GRRUser, token=token)

    # Only return if all the authorized_labels are found in the user's
    # label list, otherwise raise UnauthorizedAccess.
    if (authorized_labels.intersection(
        user.GetLabelsNames()) == authorized_labels):
      return True
    else:
      raise access_control.UnauthorizedAccess(
          "User %s is missing labels (required: %s)." % (username,
                                                         authorized_labels))
  except IOError:
    raise access_control.UnauthorizedAccess("User %s not found." % username)

Response: Verify that the username has all the authorized_labels set.


Instruction:
Explain what the following Python 3 code does
def CheckFlowCanBeStartedOnClient(flow_name):
  flow_cls = flow.GRRFlow.GetPlugin(flow_name)

  if flow_cls.category:
    return True
  else:
    raise access_control.UnauthorizedAccess(
        "Flow %s can't be started on a client by non-suid users." % flow_name)

Response: Checks if a flow can be started on a particular client.


Instruction:
How would you explain what the following Python 3 function does
def Allow(self, path, require=None, *args, **kwargs):
    regex_text = fnmatch.translate(path)
    regex = re.compile(regex_text)
    self.checks.append((regex_text, regex, require, args, kwargs))

Response: This method checks if given path pattern fits the subject passed in constructor.


Instruction:
Can you tell what is the following Python 3 function doing
def CheckAccess(self, subject, token):
    subject = rdfvalue.RDFURN(subject)
    subject_str = subject.SerializeToString()

    for check_tuple in self.checks:
      regex_text, regex, require, require_args, require_kwargs = check_tuple

      match = regex.match(subject_str)
      if not match:
        continue

      if require:
        # If require() fails, it raises access_control.UnauthorizedAccess.
        require(subject, token, *require_args, **require_kwargs)

      logging.debug(
          u"Datastore access granted to %s on %s by pattern: %s "
          u"with reason: %s (require=%s, require_args=%s, "
          u"require_kwargs=%s, helper_name=%s)",
          utils.SmartUnicode(token.username), utils.SmartUnicode(subject_str),
          utils.SmartUnicode(regex_text), utils.SmartUnicode(token.reason),
          require, require_args, require_kwargs, self.helper_name)
      return True

    logging.warning("Datastore access denied to %s (no matched rules)",
                    subject_str)
    raise access_control.UnauthorizedAccess(
        "Access to %s rejected: (no matched rules)." % subject, subject=subject)

Response: Checks for access to given subject with given token.


Instruction:
Write a Python 3 function that can
check if user has access to a client under given URN.

Response:
def _HasAccessToClient(self, subject, token):
    """Checks if user has access to a client under given URN."""
    client_id, _ = rdfvalue.RDFURN(subject).Split(2)
    client_urn = rdf_client.ClientURN(client_id)

    return self.CheckClientAccess(token, client_urn)


Instruction:
Can you tell what is the following Python 3 function doing
def _IsHomeDir(self, subject, token):
    h = CheckAccessHelper("IsHomeDir")
    h.Allow("aff4:/users/%s" % token.username)
    h.Allow("aff4:/users/%s/*" % token.username)
    try:
      return h.CheckAccess(subject, token)
    except access_control.UnauthorizedAccess:
      raise access_control.UnauthorizedAccess(
          "User can only access their "
          "home directory.", subject=subject)

Response: Checks if the user can access the subject s home directory.


Instruction:
Here you have a function in Python 3, explain what it does
def _CreateReadAccessHelper(self):
    h = CheckAccessHelper("read")

    h.Allow("aff4:/")

    # In order to open directories below aff4:/users, we have to have access to
    # aff4:/users directory itself.
    h.Allow("aff4:/users")

    # User is allowed to access anything in their home dir.
    h.Allow("aff4:/users/*", self._IsHomeDir)

    # Administrators are allowed to see current set of foreman rules.
    h.Allow("aff4:/foreman", self._UserHasAdminLabel)

    # Querying is not allowed for the blob namespace. Blobs are stored by hashes
    # as filename. If the user already knows the hash, they can access the blob,
    # however they must not be allowed to query for hashes.
    h.Allow("aff4:/blobs")
    h.Allow("aff4:/blobs/*")

    # The fingerprint namespace typically points to blobs. As such, it follows
    # the same rules.
    h.Allow("aff4:/FP")
    h.Allow("aff4:/FP/*")

    # The files namespace contains hash references to all files downloaded with
    # GRR, and is extensible via Filestore objects. Users can access files for
    # which they know the hash.
    # See lib/aff4_objects/filestore.py
    h.Allow("aff4:/files")
    h.Allow("aff4:/files/*")

    # Namespace for indexes. Client index is stored there.
    h.Allow("aff4:/index")
    h.Allow("aff4:/index/*")

    # Keyword-based Client index.
    h.Allow("aff4:/client_index")
    h.Allow("aff4:/client_index/*")

    # ACL namespace contains approval objects for accessing clients and hunts.
    h.Allow("aff4:/ACL")
    h.Allow("aff4:/ACL/*")

    # stats namespace is for different statistics. For example, ClientFleetStats
    # object is stored there.
    h.Allow("aff4:/stats")
    h.Allow("aff4:/stats/*")

    # Configuration namespace used for reading drivers, python hacks etc.
    h.Allow("aff4:/config")
    h.Allow("aff4:/config/*")

    # Namespace for flows that run without a client. A lot of internal utilitiy
    # flows and cron jobs' flows will end up here.
    h.Allow("aff4:/flows")
    h.Allow("aff4:/flows/*")

    # Namespace for hunts.
    h.Allow("aff4:/hunts")
    h.Allow("aff4:/hunts/*")

    # Namespace for cron jobs.
    h.Allow("aff4:/cron")
    h.Allow("aff4:/cron/*")

    # Namespace for audit data.
    h.Allow("aff4:/audit")
    h.Allow("aff4:/audit/*")
    h.Allow("aff4:/audit/logs")
    h.Allow("aff4:/audit/logs/*")

    # Namespace for clients.
    h.Allow(self.CLIENT_URN_PATTERN)
    h.Allow(self.CLIENT_URN_PATTERN + "/*", self._HasAccessToClient)

    # Allow everyone to read the artifact store.
    h.Allow("aff4:/artifact_store")
    h.Allow("aff4:/artifact_store/*")

    return h

Response: Creates a CheckAccessHelper for controlling read access.