Instruction:
Implement a function in Python 3 to
fill the content of the process definiton node.

Response:
def make_process_node(self, process):
        """Fill the content of process definiton node.

        :param dict process: process data as given from yaml.load function
        :return: process node

        """
        name = process['name']
        slug = process['slug']
        typ = process['type']
        version = process['version']
        description = process.get('description', '')
        source_uri = process['source_uri']
        inputs = process.get('input', [])
        outputs = process.get('output', [])

        # Make process name a section title:
        section = nodes.section(ids=['process-' + slug])
        section += nodes.title(name, name)

        # Make process header:
        section += self.make_process_header(slug, typ, version, source_uri, description, inputs)

        # Make inputs section:
        container_node = nodes.container(classes=['toggle'])
        container_header = nodes.paragraph(classes=['header'])
        container_header += nodes.strong(text='Input arguments')
        container_node += container_header

        container_body = nodes.container()
        for field_schema, _, path in iterate_schema({}, inputs, ''):
            container_body += nodes.strong(text=path)
            container_body += self.make_properties_list(field_schema)

        container_node += container_body
        section += container_node

        # Make outputs section:
        container_node = nodes.container(classes=['toggle'])
        container_header = nodes.paragraph(classes=['header'])
        container_header += nodes.strong(text='Output results')
        container_node += container_header

        container_body = nodes.container()
        for field_schema, _, path in iterate_schema({}, outputs, ''):
            container_body += nodes.strong(text=path)
            container_body += self.make_properties_list(field_schema)

        container_node += container_body
        section += container_node

        return [section, addnodes.index(entries=[('single', name, 'process-' + slug, '', None)])]


Instruction:
Here you have a function in Python 3, explain what it does
def run(self):
        config = self.state.document.settings.env.config

        # Get all processes:
        processes = get_processes(config.autoprocess_process_dir, config.autoprocess_source_base_url)
        process_nodes = []

        for process in sorted(processes, key=itemgetter('name')):
            process_nodes.extend(self.make_process_node(process))

        return process_nodes

Response: Create a list of process definitions.


Instruction:
Implement a function in Python 3 to
create a category tree.

Response:
def run(self):
        """Create a category tree."""
        config = self.state.document.settings.env.config

        # Group processes by category
        processes = get_processes(config.autoprocess_process_dir, config.autoprocess_source_base_url)
        processes.sort(key=itemgetter('category'))
        categorized_processes = {k: list(g) for k, g in groupby(processes, itemgetter('category'))}

        # Build category tree
        category_sections = {'': nodes.container(ids=['categories'])}
        top_categories = []

        for category in sorted(categorized_processes.keys()):
            category_path = ''

            for category_node in category.split(':'):
                parent_category_path = category_path
                category_path += '{}:'.format(category_node)

                if category_path in category_sections:
                    continue

                category_name = category_node.capitalize()

                section = nodes.section(ids=['category-' + category_node])
                section += nodes.title(category_name, category_name)

                # Add process list
                category_key = category_path[:-1]
                if category_key in categorized_processes:
                    listnode = nodes.bullet_list()
                    section += listnode

                    for process in categorized_processes[category_key]:
                        par = nodes.paragraph()

                        node = nodes.reference('', process['name'], internal=True)
                        node['refuri'] = config.autoprocess_definitions_uri + '#process-' + process['slug']
                        node['reftitle'] = process['name']

                        par += node
                        listnode += nodes.list_item('', par)

                category_sections[parent_category_path] += section
                category_sections[category_path] = section

                if parent_category_path == '':
                    top_categories.append(section)

        # Return top sections only
        return top_categories


Instruction:
Here you have a function in Python 3, explain what it does
def run(self):
        config = self.state.document.settings.env.config

        # Group processes by category
        processes = get_processes(config.autoprocess_process_dir, config.autoprocess_source_base_url)
        processes.sort(key=itemgetter('type'))
        processes_by_types = {k: list(g) for k, g in groupby(processes, itemgetter('type'))}

        listnode = nodes.bullet_list()

        for typ in sorted(processes_by_types.keys()):
            par = nodes.paragraph()
            par += nodes.literal(typ, typ)
            par += nodes.Text(' - ')

            processes = sorted(processes_by_types[typ], key=itemgetter('name'))
            last_process = processes[-1]
            for process in processes:
                node = nodes.reference('', process['name'], internal=True)
                node['refuri'] = config.autoprocess_definitions_uri + '#process-' + process['slug']
                node['reftitle'] = process['name']
                par += node
                if process != last_process:
                    par += nodes.Text(', ')

            listnode += nodes.list_item('', par)

        return [listnode]

Response: Create a type list.


Instruction:
Can you generate a brief explanation for the following Python 3 code
async def get(self, file_stream, pid, vendor_specific=None):
        async with await self._retry_request(
            "get", ["object", pid], vendor_specific=vendor_specific
        ) as response:
            self._assert_valid_response(response)
            async for chunk_str, _ in response.content.iter_chunks():
                file_stream.write(chunk_str)

Response: MNRead. get - Retrieve the SciObj bytes and write them to a stream.


Instruction:
How would you implement a function in Python 3 that
sends an object synchronization request to the CN.

Response:
async def synchronize(self, pid, vendor_specific=None):
        """Send an object synchronization request to the CN."""
        return await self._request_pyxb(
            "post",
            ["synchronize", pid],
            {},
            mmp_dict={"pid": pid},
            vendor_specific=vendor_specific,
        )


Instruction:
Create a Python 3 function to
encode any datetime query parameters to ISO8601.

Response:
def _datetime_to_iso8601(self, query_dict):
        """Encode any datetime query parameters to ISO8601."""
        return {
            k: v if not isinstance(v, datetime.datetime) else v.isoformat()
            for k, v in list(query_dict.items())
        }


Instruction:
Can you generate a brief explanation for the following Python 3 code
def to_representation(self, value):
        value = apply_subfield_projection(self, value, deep=True)
        return super().to_representation(value)

Response: Project outgoing native value.


Instruction:
Can you generate the documentation for the following Python 3 function
def synchronizeResponse(self, pid, vendorSpecific=None):
        mmp_dict = {'pid': pid}
        return self.POST(['synchronize'], fields=mmp_dict, headers=vendorSpecific)

Response: CNRead. synchronize - POST / synchronize. pid - PID of the current session Returns True if successful False otherwise


Instruction:
How would you code a function in Python 3 to
see Also : synchronizeResponse

Response:
def synchronize(self, pid, vendorSpecific=None):
        """See Also: synchronizeResponse() Args: pid: vendorSpecific:

        Returns:

        """
        response = self.synchronizeResponse(pid, vendorSpecific)
        return self._read_boolean_response(response)


Instruction:
Explain what the following Python 3 code does
def validate_bagit_file(bagit_path):
    _assert_zip_file(bagit_path)
    bagit_zip = zipfile.ZipFile(bagit_path)
    manifest_info_list = _get_manifest_info_list(bagit_zip)
    _validate_checksums(bagit_zip, manifest_info_list)
    return True

Response: Check if a BagIt file is valid.


Instruction:
Create a Python 3 function to
create a stream containing a BagIt zip archive.

Response:
def create_bagit_stream(dir_name, payload_info_list):
    """Create a stream containing a BagIt zip archive.

    Args:
        dir_name : str
            The name of the root directory in the zip file, under which all the files
            are placed (avoids "zip bombs").

        payload_info_list: list
            List of payload_info_dict, each dict describing a file.

            - keys: pid, filename, iter, checksum, checksum_algorithm
            - If the filename is None, the pid is used for the filename.

    """
    zip_file = zipstream.ZipFile(mode='w', compression=zipstream.ZIP_DEFLATED)
    _add_path(dir_name, payload_info_list)
    payload_byte_count, payload_file_count = _add_payload_files(
        zip_file, payload_info_list
    )
    tag_info_list = _add_tag_files(
        zip_file, dir_name, payload_info_list, payload_byte_count, payload_file_count
    )
    _add_manifest_files(zip_file, dir_name, payload_info_list, tag_info_list)
    _add_tag_manifest_file(zip_file, dir_name, tag_info_list)
    return zip_file


Instruction:
Here you have a function in Python 3, explain what it does
def _add_path(dir_name, payload_info_list):
    for payload_info_dict in payload_info_list:
        file_name = payload_info_dict['filename'] or payload_info_dict['pid']
        payload_info_dict['path'] = d1_common.utils.filesystem.gen_safe_path(
            dir_name, 'data', file_name
        )

Response: Add a key with the path to each payload_info_dict.


Instruction:
Write a Python 3 script to
add the payload files to the zip.

Response:
def _add_payload_files(zip_file, payload_info_list):
    """Add the payload files to the zip."""
    payload_byte_count = 0
    payload_file_count = 0
    for payload_info_dict in payload_info_list:
        zip_file.write_iter(payload_info_dict['path'], payload_info_dict['iter'])
        payload_byte_count += payload_info_dict['iter'].size
        payload_file_count += 1
    return payload_byte_count, payload_file_count


Instruction:
Given the following Python 3 function, write the documentation
def _add_tag_files(
    zip_file, dir_name, payload_info_list, payload_byte_count, payload_file_count
):
    """Generate the tag files and add them to the zip."""
    tag_info_list = []
    _add_tag_file(zip_file, dir_name, tag_info_list, _gen_bagit_text_file_tup())
    _add_tag_file(
        zip_file,
        dir_name,
        tag_info_list,
        _gen_bag_info_file_tup(payload_byte_count, payload_file_count),
    )
    _add_tag_file(
        zip_file, dir_name, tag_info_list, _gen_pid_mapping_file_tup(payload_info_list)
    )
    return tag_info_list

Response: Generate the tag files and add them to the zip.


Instruction:
Create a Python 3 function for
generating the manifest files and add them to the zip.

Response:
def _add_manifest_files(zip_file, dir_name, payload_info_list, tag_info_list):
    """Generate the manifest files and add them to the zip."""
    for checksum_algorithm in _get_checksum_algorithm_set(payload_info_list):
        _add_tag_file(
            zip_file,
            dir_name,
            tag_info_list,
            _gen_manifest_file_tup(payload_info_list, checksum_algorithm),
        )


Instruction:
How would you explain what the following Python 3 function does
def _add_tag_manifest_file(zip_file, dir_name, tag_info_list):
    _add_tag_file(
        zip_file, dir_name, tag_info_list, _gen_tag_manifest_file_tup(tag_info_list)
    )

Response: Generate the tag manifest file and add it to the zip.


Instruction:
Explain what the following Python 3 code does
def _add_tag_file(zip_file, dir_name, tag_info_list, tag_tup):
    tag_name, tag_str = tag_tup
    tag_path = d1_common.utils.filesystem.gen_safe_path(dir_name, tag_name)
    tag_iter = _create_and_add_tag_iter(zip_file, tag_path, tag_str)
    tag_info_list.append(
        {
            'path': tag_path,
            'checksum': d1_common.checksum.calculate_checksum_on_iterator(
                tag_iter, TAG_CHECKSUM_ALGO
            ),
        }
    )

Response: Add a tag file to zip_file and record info for the tag manifest file.


Instruction:
Given the following Python 3 function, write the documentation
def pyxb_to_dict(node_list_pyxb):
    f_dict = {}
    for f_pyxb in sorted(node_list_pyxb.node, key=lambda x: x.identifier.value()):
        f_dict[f_pyxb.identifier.value()] = {
            'name': f_pyxb.name,
            'description': f_pyxb.description,
            'base_url': f_pyxb.baseURL,
            'ping': f_pyxb.ping,
            'replicate': f_pyxb.replicate,
            'synchronize': f_pyxb.synchronize,
            'type': f_pyxb.type,
            'state': f_pyxb.state,
        }
        # TODO:
        # f_pyxb.services
        # f_pyxb.synchronization
        # f_pyxb.subject
        # f_pyxb.contactSubject
        # f_pyxb.nodeReplicationPolicy,

    return f_dict

Response: Convert a node_list_pyxb to a dict.


Instruction:
Implement a function in Python 3 to
check if the user has permission to edit the object.

Response:
def has_object_permission(self, request, view, obj):
        """Check object permissions."""
        # admins can do anything
        if request.user.is_superuser:
            return True

        # `share` permission is required for editing permissions
        if 'permissions' in view.action:
            self.perms_map['POST'] = ['%(app_label)s.share_%(model_name)s']

        if view.action in ['add_data', 'remove_data']:
            self.perms_map['POST'] = ['%(app_label)s.add_%(model_name)s']

        if hasattr(view, 'get_queryset'):
            queryset = view.get_queryset()
        else:
            queryset = getattr(view, 'queryset', None)

        assert queryset is not None, (
            'Cannot apply DjangoObjectPermissions on a view that '
            'does not set `.queryset` or have a `.get_queryset()` method.'
        )

        model_cls = queryset.model
        user = request.user

        perms = self.get_required_object_permissions(request.method, model_cls)

        if not user.has_perms(perms, obj) and not AnonymousUser().has_perms(perms, obj):
            # If the user does not have permissions we need to determine if
            # they have read permissions to see 403, or not, and simply see
            # a 404 response.

            if request.method in permissions.SAFE_METHODS:
                # Read permissions already checked and failed, no need
                # to make another lookup.
                raise Http404

            read_perms = self.get_required_object_permissions('GET', model_cls)
            if not user.has_perms(read_perms, obj):
                raise Http404

            # Has read permissions.
            return False

        return True


Instruction:
Can you generate a brief explanation for the following Python 3 code
def get_active_subject_set(self, request):
        # Handle complete certificate in vendor specific extension.
        if django.conf.settings.DEBUG_GMN:
            if 'HTTP_VENDOR_INCLUDE_CERTIFICATE' in request.META:
                request.META[
                    'SSL_CLIENT_CERT'
                ] = self.pem_in_http_header_to_pem_in_string(
                    request.META['HTTP_VENDOR_INCLUDE_CERTIFICATE']
                )

        # Add subjects from any provided certificate and JWT and store them in
        # the Django request obj.
        cert_primary_str, cert_equivalent_set = d1_gmn.app.middleware.session_cert.get_subjects(
            request
        )
        jwt_subject_list = d1_gmn.app.middleware.session_jwt.validate_jwt_and_get_subject_list(
            request
        )
        primary_subject_str = cert_primary_str
        all_subjects_set = (
            cert_equivalent_set | {cert_primary_str} | set(jwt_subject_list)
        )
        if len(jwt_subject_list) == 1:
            jwt_primary_str = jwt_subject_list[0]
            if jwt_primary_str != cert_primary_str:
                if cert_primary_str == d1_common.const.SUBJECT_PUBLIC:
                    primary_subject_str = jwt_primary_str
                else:
                    logging.warning(
                        'Both a certificate and a JWT were provided and the primary '
                        'subjects differ. Using the certificate for primary subject and'
                        'the JWT as equivalent.'
                    )

        logging.info('Primary active subject: {}'.format(primary_subject_str))
        logging.info(
            'All active subjects: {}'.format(', '.join(sorted(all_subjects_set)))
        )

        # Handle list of subjects in vendor specific extension:
        if django.conf.settings.DEBUG_GMN:
            # This is added to any subjects obtained from cert and/or JWT.
            if 'HTTP_VENDOR_INCLUDE_SUBJECTS' in request.META:
                request.all_subjects_set.update(
                    request.META['HTTP_VENDOR_INCLUDE_SUBJECTS'].split('\t')
                )

        return primary_subject_str, all_subjects_set

Response: Get a set containing all subjects for which the current connection has been successfully authenticated.


Instruction:
Here you have a function in Python 3, explain what it does
def log_setup(debug_bool):

    level = logging.DEBUG if debug_bool else logging.INFO

    logging.config.dictConfig(
        {
            "version": 1,
            "disable_existing_loggers": False,
            "formatters": {
                "verbose": {
                    "format": "%(asctime)s %(levelname)-8s %(name)s %(module)s "
                    "%(process)d %(thread)d %(message)s",
                    "datefmt": "%Y-%m-%d %H:%M:%S",
                }
            },
            "handlers": {
                "console": {
                    "class": "logging.StreamHandler",
                    "formatter": "verbose",
                    "level": level,
                    "stream": "ext://sys.stdout",
                }
            },
            "loggers": {
                "": {
                    "handlers": ["console"],
                    "level": level,
                    "class": "logging.StreamHandler",
                }
            },
        }
    )

Response:
Set up logging.

    We output only to stdout. Instead of also writing to a log file, redirect stdout to
    a log file when the script is executed from cron.


Instruction:
Make a summary of the following Python 3 code
def connect(self, dsn):
        self.con = psycopg2.connect(dsn)
        self.cur = self.con.cursor(cursor_factory=psycopg2.extras.DictCursor)
        # autocommit: Disable automatic transactions
        self.con.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT)

Response: Connect to the database.


Instruction:
Implement a Python 3 function for
getting a query parameter uniformly for GET and POST requests.

Response:
def get_query_param(self, key, default=None):
        """Get query parameter uniformly for GET and POST requests."""
        value = self.request.query_params.get(key, None)
        if value is None:
            value = self.request.data.get(key, None)
        if value is None:
            value = default
        return value


Instruction:
Implement a function in Python 3 to
get combined query parameters.

Response:
def get_query_params(self):
        """Get combined query parameters (GET and POST)."""
        params = self.request.query_params.copy()
        params.update(self.request.data)
        return params


Instruction:
Create a Python 3 function for
ordering given search by the ordering parameter given in request.

Response:
def order_search(self, search):
        """Order given search by the ordering parameter given in request.

        :param search: ElasticSearch query object

        """
        ordering = self.get_query_param('ordering', self.ordering)
        if not ordering:
            return search

        sort_fields = []
        for raw_ordering in ordering.split(','):
            ordering_field = raw_ordering.lstrip('-')
            if ordering_field not in self.ordering_fields:
                raise ParseError('Ordering by `{}` is not supported.'.format(ordering_field))

            ordering_field = self.ordering_map.get(ordering_field, ordering_field)
            direction = '-' if raw_ordering[0] == '-' else ''
            sort_fields.append('{}{}'.format(direction, ordering_field))

        return search.sort(*sort_fields)


Instruction:
Write a Python 3 function for
filtering given search by the filter parameter given in request.

Response:
def filter_search(self, search):
        """Filter given search by the filter parameter given in request.

        :param search: ElasticSearch query object

        """
        builder = QueryBuilder(
            self.filtering_fields,
            self.filtering_map,
            self
        )
        search, unmatched = builder.build(search, self.get_query_params())

        # Ensure that no unsupported arguments were used.
        for argument in self.get_always_allowed_arguments():
            unmatched.pop(argument, None)

        if unmatched:
            msg = 'Unsupported parameter(s): {}. Please use a combination of: {}.'.format(
                ', '.join(unmatched),
                ', '.join(self.filtering_fields),
            )
            raise ParseError(msg)

        return search


Instruction:
Can you create a Python 3 function that
filters given query based on permissions of the user in the request.

Response:
def filter_permissions(self, search):
        """Filter given query based on permissions of the user in the request.

        :param search: ElasticSearch query object

        """
        user = self.request.user
        if user.is_superuser:
            return search
        if user.is_anonymous:
            user = get_anonymous_user()

        filters = [Q('match', users_with_permissions=user.pk)]
        filters.extend([
            Q('match', groups_with_permissions=group.pk) for group in user.groups.all()
        ])
        filters.append(Q('match', public_permission=True))

        # `minimum_should_match` is set to 1 by default
        return search.query('bool', should=filters)


Instruction:
Can you generate the documentation for the following Python 3 function
def paginate_response(self, queryset, serializers_kwargs={}):
        page = self.paginate_queryset(queryset)
        if page is not None:
            serializer = self.get_serializer(page, many=True, **serializers_kwargs)
            return self.get_paginated_response(serializer.data)

        serializer = self.get_serializer(queryset, many=True, **serializers_kwargs)
        return Response(serializer.data)

Response: Optionally return paginated response.


Instruction:
Make a summary of the following Python 3 code
def search(self):
        search = self.document_class().search()  # pylint: disable=not-callable

        search = self.custom_filter(search)

        search = self.filter_search(search)
        search = self.order_search(search)
        search = self.filter_permissions(search)

        if search.count() > ELASTICSEARCH_SIZE:
            limit = self.paginator.get_limit(self.request)

            if not limit or limit > ELASTICSEARCH_SIZE:
                raise TooManyResults()

        search = search.extra(size=ELASTICSEARCH_SIZE)
        return search

Response: Handle the search request.


Instruction:
Write a Python 3 function for
calling by FUSE when the attributes for a file or directory are required.

Response:
def getattr(self, path, fh):
        """Called by FUSE when the attributes for a file or directory are required.

        Returns a dictionary with keys identical to the stat C structure of stat(2).
        st_atime, st_mtime and st_ctime should be floats. On OSX, st_nlink should count
        all files inside the directory. On Linux, only the subdirectories are counted.
        The 'st_dev' and 'st_blksize' fields are ignored. The 'st_ino' field is ignored
        except if the 'use_ino' mount option is given.

        This method gets very heavy traffic.

        """
        self._raise_error_if_os_special_file(path)
        # log.debug(u'getattr(): {0}'.format(path))
        attribute = self._get_attributes_through_cache(path)
        # log.debug('getattr() returned attribute: {0}'.format(attribute))
        return self._stat_from_attributes(attribute)


Instruction:
Write a Python 3 script to
call by FUSE when a directory is opened. Returns a list of file and directory names for the directory.

Response:
def readdir(self, path, fh):
        """Called by FUSE when a directory is opened.

        Returns a list of file and directory names for the directory.

        """
        log.debug('readdir(): {}'.format(path))
        try:
            dir = self._directory_cache[path]
        except KeyError:
            dir = self._get_directory(path)
            self._directory_cache[path] = dir
        return dir


Instruction:
Create a Python 3 function to
call by FUSE when a file is opened.

Response:
def open(self, path, flags):
        """Called by FUSE when a file is opened.

        Determines if the provided path and open flags are valid.

        """
        log.debug('open(): {}'.format(path))
        # ONEDrive is currently read only. Anything but read access is denied.
        if (flags & self._READ_ONLY_ACCESS_MODE) != os.O_RDONLY:
            self._raise_error_permission_denied(path)
        # Any file in the filesystem can be opened.
        attribute = self._get_attributes_through_cache(path)
        return attribute.is_dir()


Instruction:
Write a Python 3 script for
extracting an inline expression from the given text.

Response:
def get_inline_expression(self, text):
        """Extract an inline expression from the given text."""
        text = text.strip()
        if not text.startswith(self.inline_tags[0]) or not text.endswith(self.inline_tags[1]):
            return

        return text[2:-2]


Instruction:
Implement a Python 3 function for
filtering queryset by entity label and position.

Response:
def _filter_queryset(self, queryset):
        """Filter queryset by entity, label and position.

        Due to a bug in django-filter these filters have to be applied
        manually:
        https://github.com/carltongibson/django-filter/issues/883
        """
        entities = self.request.query_params.getlist('entity')
        labels = self.request.query_params.getlist('label')
        positions = self.request.query_params.getlist('position')

        if labels and len(labels) != len(entities):
            raise exceptions.ParseError(
                'If `labels` query parameter is given, also `entities` '
                'must be given and they must be of the same length.'
            )

        if positions and len(positions) != len(entities):
            raise exceptions.ParseError(
                'If `positions` query parameter is given, also `entities` '
                'must be given and they must be of the same length.'
            )

        if entities:
            for entity, label, position in zip_longest(entities, labels, positions):
                filter_params = {'entities__pk': entity}
                if label:
                    filter_params['relationpartition__label'] = label
                if position:
                    filter_params['relationpartition__position'] = position

                queryset = queryset.filter(**filter_params)

        return queryset


Instruction:
Make a summary of the following Python 3 code
def update(self, request, *args, **kwargs):
        instance = self.get_object()
        if (not request.user.has_perm('edit_collection', instance.collection)
                and not request.user.is_superuser):
            return Response(status=status.HTTP_401_UNAUTHORIZED)

        return super().update(request, *args, **kwargs)

Response: Update the relation object.


Instruction:
Write a Python 3 script for
rebuilding indexes affected by the given permission.

Response:
def _process_permission(perm):
    """Rebuild indexes affected by the given permission."""
    # XXX: Optimize: rebuild only permissions, not whole document
    codename = perm.permission.codename
    if not codename.startswith('view') and not codename.startswith('owner'):
        return

    index_builder.build(perm.content_object)


Instruction:
Explain what the following Python 3 code does
def parse_response(response, encoding='utf-8'):
    return requests_toolbelt.multipart.decoder.MultipartDecoder.from_response(
        response, encoding
    ).parts

Response: Parse a multipart Requests. Response into a tuple of BodyPart objects.


Instruction:
Implement a Python 3 function for
parsing a string representation of a multipart document into a tuple of BodyPart objects.

Response:
def parse_str(mmp_bytes, content_type, encoding='utf-8'):
    """Parse multipart document bytes into a tuple of BodyPart objects.

    Args:
      mmp_bytes: bytes
        Multipart document.

      content_type : str
        Must be on the form, ``multipart/form-data; boundary=<BOUNDARY>``, where
        ``<BOUNDARY>`` is the string that separates the parts of the multipart document
        in ``mmp_bytes``. In HTTP requests and responses, it is passed in the
        Content-Type header.

      encoding : str
        The coding used for the text in the HTML body.

    Returns:
      tuple of BodyPart
        Members: headers (CaseInsensitiveDict), content (bytes), text (Unicode),
        encoding (str).

    """
    return requests_toolbelt.multipart.decoder.MultipartDecoder(
        mmp_bytes, content_type, encoding
    ).parts


Instruction:
How would you implement a function in Python 3 that
normalizes a tuple of BodyPart objects to a string.

Response:
def normalize(body_part_tup,):
    """Normalize a tuple of BodyPart objects to a string.

    Normalization is done by sorting the body_parts by the Content- Disposition headers,
    which is typically on the form, ``form-data; name="name_of_part``.

    """
    return '\n\n'.join(
        [
            '{}\n\n{}'.format(
                str(p.headers[b'Content-Disposition'], p.encoding), p.text
            )
            for p in sorted(
                body_part_tup, key=lambda p: p.headers[b'Content-Disposition']
            )
        ]
    )


Instruction:
Explain what the following Python 3 code does
def is_multipart(header_dict):
    return (
        {k.lower(): v for k, v in header_dict.items()}
        .get('content-type', '')
        .startswith('multipart')
    )

Response: Checks if the given dict contains a Content - Type key that starts with multipart.


Instruction:
Can you generate the documentation for the following Python 3 function
def _split_path_by_reserved_name(self, path):
        for i, e in enumerate(path):
            if e in self._resolvers or e == self._get_readme_filename():
                return path[:i], path[i], path[i + 1 :]
        raise d1_onedrive.impl.onedrive_exceptions.PathException(
            'Invalid folder: %s' % str(path)
        )

Response: Split the path by reserved name.


Instruction:
Write a Python 3 function that can
generate a human readable description of the folder in text format.

Response:
def _generate_readme_text(self, object_tree_path):
        """Generate a human readable description of the folder in text format."""

        wdef_folder = self._object_tree.get_source_tree_folder(object_tree_path)
        res = StringIO()
        if len(object_tree_path):
            folder_name = object_tree_path[-1]
        else:
            folder_name = 'root'
        header = 'ObjectTree Folder "{}"'.format(folder_name)
        res.write(header + '\n')
        res.write('{}\n\n'.format('=' * len(header)))
        res.write(
            'The content present in object_tree folders is determined by a list\n'
        )
        res.write(
            'of specific identifiers and by queries applied against the DataONE\n'
        )
        res.write('search index.\n\n')
        res.write('Queries:\n\n')
        if len(wdef_folder['queries']):
            for query in wdef_folder['queries']:
                res.write('- {}\n'.format(query))
        else:
            res.write('No queries specified at this level.\n')
        res.write('\n\n')
        res.write('Identifiers:\n\n')
        if len(wdef_folder['identifiers']):
            for pid in wdef_folder['identifiers']:
                res.write('- {}\n'.format(pid))
        else:
            res.write('No individual identifiers selected at this level.\n')
        res.write('\n\n')
        res.write('Sub-folders:\n\n')
        if len(wdef_folder['collections']):
            for f in wdef_folder['collections']:
                res.write('- {}\n'.format(f))
        else:
            res.write('No object_tree sub-folders are specified at this level.\n')
        return res.getvalue().encode('utf-8')


Instruction:
Write a Python 3 function for
returning serialized data or list of ids depending on hydrate_data query param.

Response:
def _serialize_data(self, data):
        """Return serialized data or list of ids, depending on `hydrate_data` query param."""
        if self.request and self.request.query_params.get('hydrate_data', False):
            serializer = DataSerializer(data, many=True, read_only=True)
            serializer.bind('data', self)
            return serializer.data
        else:
            return [d.id for d in data]


Instruction:
Create a Python 3 function to
filter object objects by permissions of user in request.

Response:
def _filter_queryset(self, perms, queryset):
        """Filter object objects by permissions of user in request."""
        user = self.request.user if self.request else AnonymousUser()
        return get_objects_for_user(user, perms, queryset)


Instruction:
Here you have a function in Python 3, explain what it does
def get_data(self, collection):
        data = self._filter_queryset('view_data', collection.data.all())

        return self._serialize_data(data)

Response: Return serialized list of data objects on collection that user has view permission on.


Instruction:
Given the following Python 3 function, write the documentation
def get_fields(self):
        fields = super(CollectionSerializer, self).get_fields()

        if self.request.method == "GET":
            fields['data'] = serializers.SerializerMethodField()
        else:
            fields['data'] = serializers.PrimaryKeyRelatedField(many=True, read_only=True)

        return fields

Response: Dynamically adapt fields based on the current request.


Instruction:
Can you create a Python 3 function that
returns True if sysmeta_pyxb is a SystemMetadata PyXB object.

Response:
def is_sysmeta_pyxb(sysmeta_pyxb):
    """Args: sysmeta_pyxb: Object that may or may not be a SystemMetadata PyXB object.

    Returns:
      bool:
        - ``True`` if ``sysmeta_pyxb`` is a SystemMetadata PyXB object.
        - ``False`` if ``sysmeta_pyxb`` is not a PyXB object or is a PyXB object of a
          type other than SystemMetadata.

    """
    return (
        d1_common.type_conversions.is_pyxb_d1_type(sysmeta_pyxb)
        and d1_common.type_conversions.pyxb_get_type_name(sysmeta_pyxb)
        == 'SystemMetadata'
    )


Instruction:
Can you generate the documentation for the following Python 3 function
def normalize_in_place(sysmeta_pyxb, reset_timestamps=False):
    if sysmeta_pyxb.accessPolicy is not None:
        sysmeta_pyxb.accessPolicy = d1_common.wrap.access_policy.get_normalized_pyxb(
            sysmeta_pyxb.accessPolicy
        )
    if getattr(sysmeta_pyxb, 'mediaType', False):
        d1_common.xml.sort_value_list_pyxb(sysmeta_pyxb.mediaType.property_)
    if getattr(sysmeta_pyxb, 'replicationPolicy', False):
        d1_common.xml.sort_value_list_pyxb(
            sysmeta_pyxb.replicationPolicy.preferredMemberNode
        )
        d1_common.xml.sort_value_list_pyxb(
            sysmeta_pyxb.replicationPolicy.blockedMemberNode
        )
    d1_common.xml.sort_elements_by_child_values(
        sysmeta_pyxb.replica,
        ['replicaVerified', 'replicaMemberNode', 'replicationStatus'],
    )
    sysmeta_pyxb.archived = bool(sysmeta_pyxb.archived)
    if reset_timestamps:
        epoch_dt = datetime.datetime(1970, 1, 1, tzinfo=d1_common.date_time.UTC())
        sysmeta_pyxb.dateUploaded = epoch_dt
        sysmeta_pyxb.dateSysMetadataModified = epoch_dt
        for replica_pyxb in getattr(sysmeta_pyxb, 'replica', []):
            replica_pyxb.replicaVerified = epoch_dt
    else:
        sysmeta_pyxb.dateUploaded = d1_common.date_time.round_to_nearest(
            sysmeta_pyxb.dateUploaded
        )
        sysmeta_pyxb.dateSysMetadataModified = d1_common.date_time.round_to_nearest(
            sysmeta_pyxb.dateSysMetadataModified
        )
        for replica_pyxb in getattr(sysmeta_pyxb, 'replica', []):
            replica_pyxb.replicaVerified = d1_common.date_time.round_to_nearest(
                replica_pyxb.replicaVerified
            )

Response: Normalizes SystemMetadata PyXB object in - place.


Instruction:
Make a summary of the following Python 3 code
def are_equivalent_pyxb(a_pyxb, b_pyxb, ignore_timestamps=False):
    normalize_in_place(a_pyxb, ignore_timestamps)
    normalize_in_place(b_pyxb, ignore_timestamps)
    a_xml = d1_common.xml.serialize_to_xml_str(a_pyxb)
    b_xml = d1_common.xml.serialize_to_xml_str(b_pyxb)
    are_equivalent = d1_common.xml.are_equivalent(a_xml, b_xml)
    if not are_equivalent:
        logger.debug('XML documents not equivalent:')
        logger.debug(d1_common.xml.format_diff_xml(a_xml, b_xml))
    return are_equivalent

Response: Determine if two SystemMetadata PyXB objects are semantically equivalent.


Instruction:
How would you implement a function in Python 3 that
determines if two SystemMetadata XML docs are semantically equivalent.

Response:
def are_equivalent_xml(a_xml, b_xml, ignore_timestamps=False):
    """Determine if two SystemMetadata XML docs are semantically equivalent.

    Normalize then compare SystemMetadata XML docs for equivalency.

    Args:
      a_xml, b_xml: bytes
        UTF-8 encoded SystemMetadata XML docs to compare

      ignore_timestamps: bool
        ``True``: Timestamps in the SystemMetadata are ignored so that objects that are
        compared register as equivalent if only their timestamps differ.

    Returns:
      bool: **True** if SystemMetadata XML docs are semantically equivalent.

    Notes:
      The SystemMetadata is normalized by removing any redundant information and
      ordering all sections where there are no semantics associated with the order. The
      normalized SystemMetadata is intended to be semantically equivalent to the
      un-normalized one.

    """

    """Normalizes then compares SystemMetadata XML docs for equivalency.
  ``a_xml`` and ``b_xml`` should be utf-8 encoded DataONE System Metadata XML
  documents.
  """
    return are_equivalent_pyxb(
        d1_common.xml.deserialize(a_xml),
        d1_common.xml.deserialize(b_xml),
        ignore_timestamps,
    )


Instruction:
Can you implement a function in Python 3 that
clears the sysmeta_pyxb s elements.

Response:
def clear_elements(sysmeta_pyxb, clear_replica=True, clear_serial_version=True):
    """{clear_replica} causes any replica information to be removed from the object.

    {clear_replica} ignores any differences in replica information, as this information
    is often different between MN and CN.

    """
    if clear_replica:
        sysmeta_pyxb.replica = None
    if clear_serial_version:
        sysmeta_pyxb.serialVersion = None

    sysmeta_pyxb.replicationPolicy = None


Instruction:
Can you create a Python 3 function that
copies elements specified in el_list from src_pyxb to dst_pyxb.

Response:
def update_elements(dst_pyxb, src_pyxb, el_list):
    """Copy elements specified in ``el_list`` from ``src_pyxb`` to ``dst_pyxb``

    Only elements that are children of root are supported. See
    SYSMETA_ROOT_CHILD_LIST.

    If an element in ``el_list`` does not exist in ``src_pyxb``, it is removed from
    ``dst_pyxb``.

    """
    invalid_element_set = set(el_list) - set(SYSMETA_ROOT_CHILD_LIST)
    if invalid_element_set:
        raise ValueError(
            'Passed one or more invalid elements. invalid="{}"'.format(
                ', '.join(sorted(list(invalid_element_set)))
            )
        )
    for el_str in el_list:
        setattr(dst_pyxb, el_str, getattr(src_pyxb, el_str, None))


Instruction:
Implement a Python 3 function for
joining action with the content type of obj.

Response:
def get_full_perm(perm, obj):
    """Join action with the content type of ``obj``.

    Permission is returned in the format of ``<action>_<object_type>``.
    """
    ctype = ContentType.objects.get_for_model(obj)
    # Camel case class names are converted into a space-separated
    # content types, so spaces have to be removed.
    ctype = str(ctype).replace(' ', '')

    return '{}_{}'.format(perm.lower(), ctype)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def copy_permissions(src_obj, dest_obj):
    def _process_permission(codename, user_or_group, dest_obj, relabel):
        """Process single permission."""
        if relabel:
            codename = change_perm_ctype(codename, dest_obj)
            if codename not in dest_all_perms:
                return  # dest object doesn't have matching permission

        assign_perm(codename, user_or_group, dest_obj)

    src_obj_ctype = ContentType.objects.get_for_model(src_obj)
    dest_obj_ctype = ContentType.objects.get_for_model(dest_obj)
    dest_all_perms = get_all_perms(dest_obj)

    relabel = (src_obj_ctype != dest_obj_ctype)

    for perm in UserObjectPermission.objects.filter(object_pk=src_obj.pk, content_type=src_obj_ctype):
        _process_permission(perm.permission.codename, perm.user, dest_obj, relabel)

    for perm in GroupObjectPermission.objects.filter(object_pk=src_obj.pk, content_type=src_obj_ctype):
        _process_permission(perm.permission.codename, perm.group, dest_obj, relabel)

Response: Copy permissions from src_obj to dest_obj.


Instruction:
Write a Python 3 function that can
get user by pk or username. Raise error if it doesn t exist.

Response:
def fetch_user(query):
    """Get user by ``pk`` or ``username``. Raise error if it doesn't exist."""
    user_filter = {'pk': query} if query.isdigit() else {'username': query}
    user_model = get_user_model()

    try:
        return user_model.objects.get(**user_filter)
    except user_model.DoesNotExist:
        raise exceptions.ParseError("Unknown user: {}".format(query))


Instruction:
Write a Python 3 function for
getting group by pk or name. Raise error if it doesn t exist.

Response:
def fetch_group(query):
    """Get group by ``pk`` or ``name``. Raise error if it doesn't exist."""
    group_filter = {'pk': query} if query.isdigit() else {'name': query}

    try:
        return Group.objects.get(**group_filter)
    except Group.DoesNotExist:
        raise exceptions.ParseError("Unknown group: {}".format(query))


Instruction:
Given the following Python 3 function, write the documentation
def check_owner_permission(payload, allow_user_owner):
    for entity_type in ['users', 'groups']:
        for perm_type in ['add', 'remove']:
            for perms in payload.get(entity_type, {}).get(perm_type, {}).values():
                if 'owner' in perms:
                    if entity_type == 'users' and allow_user_owner:
                        continue

                    if entity_type == 'groups':
                        raise exceptions.ParseError("Owner permission cannot be assigned to a group")

                    raise exceptions.PermissionDenied("Only owners can grant/revoke owner permission")

Response: Raise PermissionDenied if owner is not allowed.


Instruction:
Given the following Python 3 function, write the documentation
def check_public_permissions(payload):
    allowed_public_permissions = ['view', 'add', 'download']
    for perm_type in ['add', 'remove']:
        for perm in payload.get('public', {}).get(perm_type, []):
            if perm not in allowed_public_permissions:
                raise exceptions.PermissionDenied("Permissions for public users are too open")

Response: Raise PermissionDenied if public permissions are too open.


Instruction:
Write a Python 3 function for
raising PermissionDenied if user_pk is not in the payload.

Response:
def check_user_permissions(payload, user_pk):
    """Raise ``PermissionDenied`` if ``payload`` includes ``user_pk``."""
    for perm_type in ['add', 'remove']:
        user_pks = payload.get('users', {}).get(perm_type, {}).keys()
        if user_pk in user_pks:
            raise exceptions.PermissionDenied("You cannot change your own permissions")


Instruction:
Write a Python 3 function that can
remove all occurrences of permission from payload.

Response:
def remove_permission(payload, permission):
    """Remove all occurrences of ``permission`` from ``payload``."""
    payload = copy.deepcopy(payload)

    for entity_type in ['users', 'groups']:
        for perm_type in ['add', 'remove']:
            for perms in payload.get(entity_type, {}).get(perm_type, {}).values():
                if permission in perms:
                    perms.remove(permission)

    for perm_type in ['add', 'remove']:
        perms = payload.get('public', {}).get(perm_type, [])
        if permission in perms:
            perms.remove(permission)

    return payload


Instruction:
How would you explain what the following Python 3 function does
def update_permission(obj, data):
    full_permissions = get_all_perms(obj)

    def apply_perm(perm_func, perms, entity):
        """Apply permissions using given ``perm_func``.

        ``perm_func`` is intended to be ``assign_perms`` or
        ``remove_perms`` shortcut function from ``django-guardian``, but
        can be any function that accepts permission codename,
        user/group and object parameters (in this order).

        If given permission does not exist, ``exceptions.ParseError`` is
        raised.

        "ALL" passed as ``perms`` parameter, will call ``perm_function``
        with ``full_permissions`` list.

        :param func perm_func: Permissions function to be applied
        :param list params: list of params to be allpied
        :param entity: user or group to be passed to ``perm_func``
        :type entity: `~django.contrib.auth.models.User` or
            `~django.contrib.auth.models.Group`

        """
        if perms == 'ALL':
            perms = full_permissions
        for perm in perms:
            perm_codename = get_full_perm(perm, obj)
            if perm_codename not in full_permissions:
                raise exceptions.ParseError("Unknown permission: {}".format(perm))
            perm_func(perm_codename, entity, obj)

    def set_permissions(entity_type, perm_type):
        """Set object permissions."""
        perm_func = assign_perm if perm_type == 'add' else remove_perm
        fetch_fn = fetch_user if entity_type == 'users' else fetch_group

        for entity_id in data.get(entity_type, {}).get(perm_type, []):
            entity = fetch_fn(entity_id)
            if entity:
                perms = data[entity_type][perm_type][entity_id]
                apply_perm(perm_func, perms, entity)

    def set_public_permissions(perm_type):
        """Set public permissions."""
        perm_func = assign_perm if perm_type == 'add' else remove_perm
        user = AnonymousUser()
        perms = data.get('public', {}).get(perm_type, [])
        apply_perm(perm_func, perms, user)

    with transaction.atomic():
        set_permissions('users', 'add')
        set_permissions('users', 'remove')
        set_permissions('groups', 'add')
        set_permissions('groups', 'remove')
        set_public_permissions('add')
        set_public_permissions('remove')

Response: Update object s permissions using given data.


Instruction:
Explain what the following Python 3 code does
def assign_contributor_permissions(obj, contributor=None):
    for permission in get_all_perms(obj):
        assign_perm(permission, contributor if contributor else obj.contributor, obj)

Response: Assign all permissions to object s contributor.


Instruction:
Can you implement a function in Python 3 that
executes the script and save results.

Response:
async def run_script(self, script):
        """Execute the script and save results."""
        script = os.linesep.join(['set -x', 'set +B', script, 'exit']) + os.linesep
        self.proc.stdin.write(script.encode('utf-8'))
        await self.proc.stdin.drain()
        self.proc.stdin.close()


Instruction:
Here you have a function in Python 3, explain what it does
async def terminate(self):
        self.proc.terminate()

        await asyncio.wait_for(self.proc.wait(), self.kill_delay)
        if self.proc.returncode is None:
            self.proc.kill()
        await self.proc.wait()

        await super().terminate()

Response: Terminate a running script.


Instruction:
How would you implement a function in Python 3 that
constructs a connection to Redis.

Response:
async def _make_connection(self):
        """Construct a connection to Redis."""
        return await aioredis.create_redis(
            'redis://{}:{}'.format(
                self._redis_params.get('host', 'localhost'),
                self._redis_params.get('port', 6379)
            ),
            db=int(self._redis_params.get('db', 1))
        )


Instruction:
Can you create a Python 3 function that
performs a Redis call and handle connection dropping.

Response:
async def _call_redis(self, meth, *args, **kwargs):
        """Perform a Redis call and handle connection dropping."""
        while True:
            try:
                if not self._redis:
                    self._redis = await self._make_connection()
                return await meth(self._redis, *args, **kwargs)
            except aioredis.RedisError:
                logger.exception("Redis connection error")
                if self._redis:
                    self._redis.close()
                    await self._redis.wait_closed()
                    self._redis = None
                await asyncio.sleep(3)


Instruction:
Can you implement a function in Python 3 that
clears the executor queue channel.

Response:
async def clear_queue(self):
        """Reset the executor queue channel to an empty state."""
        conn = await self._make_connection()
        try:
            script = """
                local keys = redis.call('KEYS', ARGV[1])
                redis.call('DEL', unpack(keys))
            """
            await conn.eval(
                script,
                keys=[],
                args=['*{}*'.format(settings.FLOW_MANAGER['REDIS_PREFIX'])],
            )
        finally:
            conn.close()


Instruction:
Can you generate a brief explanation for the following Python 3 code
def terminate(self):
        logger.info(__(
            "Terminating Resolwe listener on channel '{}'.",
            state.MANAGER_EXECUTOR_CHANNELS.queue
        ))
        self._should_stop = True

Response: Stop the standalone manager.


Instruction:
Create a Python 3 function to
generate the feedback channel name from the object s id.

Response:
def _queue_response_channel(self, obj):
        """Generate the feedback channel name from the object's id.

        :param obj: The Channels message object.
        """
        return '{}.{}'.format(state.MANAGER_EXECUTOR_CHANNELS.queue_response, obj[ExecutorProtocol.DATA_ID])


Instruction:
Write a Python 3 script to
send a reply to the executor.

Response:
async def _send_reply(self, obj, reply):
        """Send a reply with added standard fields back to executor.

        :param obj: The original Channels message object to which we're
            replying.
        :param reply: The message contents dictionary. The data id is
            added automatically (``reply`` is modified in place).
        """
        reply.update({
            ExecutorProtocol.DATA_ID: obj[ExecutorProtocol.DATA_ID],
        })
        await self._call_redis(aioredis.Redis.rpush, self._queue_response_channel(obj), json.dumps(reply))


Instruction:
Write a Python 3 script to
pop the given file s map from the exported files mapping.

Response:
def hydrate_spawned_files(self, exported_files_mapper, filename, data_id):
        """Pop the given file's map from the exported files mapping.

        :param exported_files_mapper: The dict of file mappings this
            process produced.
        :param filename: The filename to format and remove from the
            mapping.
        :param data_id: The id of the :meth:`~resolwe.flow.models.Data`
            object owning the mapping.
        :return: The formatted mapping between the filename and
            temporary file path.
        :rtype: dict
        """
        # JSON only has string dictionary keys, so the Data object id
        # needs to be stringified first.
        data_id = str(data_id)

        if filename not in exported_files_mapper[data_id]:
            raise KeyError("Use 're-export' to prepare the file for spawned process: {}".format(filename))

        export_fn = exported_files_mapper[data_id].pop(filename)

        if exported_files_mapper[data_id] == {}:
            exported_files_mapper.pop(data_id)

        return {'file_temp': export_fn, 'file': filename}


Instruction:
Here you have a function in Python 3, explain what it does
def handle_update(self, obj, internal_call=False):
        data_id = obj[ExecutorProtocol.DATA_ID]
        changeset = obj[ExecutorProtocol.UPDATE_CHANGESET]
        if not internal_call:
            logger.debug(
                __("Handling update for Data with id {} (handle_update).", data_id),
                extra={
                    'data_id': data_id,
                    'packet': obj
                }
            )
        try:
            d = Data.objects.get(pk=data_id)
        except Data.DoesNotExist:
            logger.warning(
                "Data object does not exist (handle_update).",
                extra={
                    'data_id': data_id,
                }
            )

            if not internal_call:
                async_to_sync(self._send_reply)(obj, {ExecutorProtocol.RESULT: ExecutorProtocol.RESULT_ERROR})

            async_to_sync(consumer.send_event)({
                WorkerProtocol.COMMAND: WorkerProtocol.ABORT,
                WorkerProtocol.DATA_ID: obj[ExecutorProtocol.DATA_ID],
                WorkerProtocol.FINISH_COMMUNICATE_EXTRA: {
                    'executor': getattr(settings, 'FLOW_EXECUTOR', {}).get('NAME', 'resolwe.flow.executors.local'),
                },
            })

            return

        if changeset.get('status', None) == Data.STATUS_ERROR:
            logger.error(
                __("Error occured while running process '{}' (handle_update).", d.process.slug),
                extra={
                    'data_id': data_id,
                    'api_url': '{}{}'.format(
                        getattr(settings, 'RESOLWE_HOST_URL', ''),
                        reverse('resolwe-api:data-detail', kwargs={'pk': data_id})
                    ),
                }
            )

        if d.status == Data.STATUS_ERROR:
            changeset['status'] = Data.STATUS_ERROR

        if not d.started:
            changeset['started'] = now()
        changeset['modified'] = now()

        for key, val in changeset.items():
            if key in ['process_error', 'process_warning', 'process_info']:
                # Trim process_* fields to not exceed max length of the database field.
                for i, entry in enumerate(val):
                    max_length = Data._meta.get_field(key).base_field.max_length  # pylint: disable=protected-access
                    if len(entry) > max_length:
                        val[i] = entry[:max_length - 3] + '...'

                getattr(d, key).extend(val)

            elif key != 'output':
                setattr(d, key, val)

        if 'output' in changeset:
            if not isinstance(d.output, dict):
                d.output = {}
            for key, val in changeset['output'].items():
                dict_dot(d.output, key, val)

        try:
            d.save(update_fields=list(changeset.keys()))
        except ValidationError as exc:
            logger.error(
                __(
                    "Validation error when saving Data object of process '{}' (handle_update):\n\n{}",
                    d.process.slug,
                    traceback.format_exc()
                ),
                extra={
                    'data_id': data_id
                }
            )

            d.refresh_from_db()

            d.process_error.append(exc.message)
            d.status = Data.STATUS_ERROR

            try:
                d.save(update_fields=['process_error', 'status'])
            except Exception:  # pylint: disable=broad-except
                pass
        except Exception:  # pylint: disable=broad-except
            logger.error(
                __(
                    "Error when saving Data object of process '{}' (handle_update):\n\n{}",
                    d.process.slug,
                    traceback.format_exc()
                ),
                extra={
                    'data_id': data_id
                }
            )

        if not internal_call:
            async_to_sync(self._send_reply)(obj, {ExecutorProtocol.RESULT: ExecutorProtocol.RESULT_OK})

Response: Handles an incoming update request.


Instruction:
Can you create a Python 3 function that
handles an incoming Data finished processing request.

Response:
def handle_finish(self, obj):
        """Handle an incoming ``Data`` finished processing request.

        :param obj: The Channels message object. Command object format:

            .. code-block:: none

                {
                    'command': 'finish',
                    'data_id': [id of the :class:`~resolwe.flow.models.Data` object
                               this command changes],
                    'process_rc': [exit status of the processing]
                    'spawn_processes': [optional; list of spawn dictionaries],
                    'exported_files_mapper': [if spawn_processes present]
                }
        """
        data_id = obj[ExecutorProtocol.DATA_ID]
        logger.debug(
            __("Finishing Data with id {} (handle_finish).", data_id),
            extra={
                'data_id': data_id,
                'packet': obj
            }
        )
        spawning_failed = False
        with transaction.atomic():
            # Spawn any new jobs in the request.
            spawned = False
            if ExecutorProtocol.FINISH_SPAWN_PROCESSES in obj:
                if is_testing():
                    # NOTE: This is a work-around for Django issue #10827
                    # (https://code.djangoproject.com/ticket/10827), same as in
                    # TestCaseHelpers._pre_setup(). Because the listener is running
                    # independently, it must clear the cache on its own.
                    ContentType.objects.clear_cache()

                spawned = True
                exported_files_mapper = obj[ExecutorProtocol.FINISH_EXPORTED_FILES]
                logger.debug(
                    __("Spawning new Data objects for Data with id {} (handle_finish).", data_id),
                    extra={
                        'data_id': data_id
                    }
                )

                try:
                    # This transaction is needed because we're running
                    # asynchronously with respect to the main Django code
                    # here; the manager can get nudged from elsewhere.
                    with transaction.atomic():
                        parent_data = Data.objects.get(pk=data_id)

                        # Spawn processes.
                        for d in obj[ExecutorProtocol.FINISH_SPAWN_PROCESSES]:
                            d['contributor'] = parent_data.contributor
                            d['process'] = Process.objects.filter(slug=d['process']).latest()
                            d['tags'] = parent_data.tags

                            for field_schema, fields in iterate_fields(d.get('input', {}), d['process'].input_schema):
                                type_ = field_schema['type']
                                name = field_schema['name']
                                value = fields[name]

                                if type_ == 'basic:file:':
                                    fields[name] = self.hydrate_spawned_files(
                                        exported_files_mapper, value, data_id
                                    )
                                elif type_ == 'list:basic:file:':
                                    fields[name] = [self.hydrate_spawned_files(exported_files_mapper, fn, data_id)
                                                    for fn in value]

                            with transaction.atomic():
                                d = Data.objects.create(**d)
                                DataDependency.objects.create(
                                    parent=parent_data,
                                    child=d,
                                    kind=DataDependency.KIND_SUBPROCESS,
                                )

                                # Copy permissions.
                                copy_permissions(parent_data, d)

                                # Entity is added to the collection only when it is
                                # created - when it only contains 1 Data object.
                                entities = Entity.objects.filter(data=d).annotate(num_data=Count('data')).filter(
                                    num_data=1)

                                # Copy collections.
                                for collection in parent_data.collection_set.all():
                                    collection.data.add(d)

                                    # Add entities to which data belongs to the collection.
                                    for entity in entities:
                                        entity.collections.add(collection)

                except Exception:  # pylint: disable=broad-except
                    logger.error(
                        __(
                            "Error while preparing spawned Data objects of process '{}' (handle_finish):\n\n{}",
                            parent_data.process.slug,
                            traceback.format_exc()
                        ),
                        extra={
                            'data_id': data_id
                        }
                    )
                    spawning_failed = True

            # Data wrap up happens last, so that any triggered signals
            # already see the spawned children. What the children themselves
            # see is guaranteed by the transaction we're in.
            if ExecutorProtocol.FINISH_PROCESS_RC in obj:
                process_rc = obj[ExecutorProtocol.FINISH_PROCESS_RC]

                try:
                    d = Data.objects.get(pk=data_id)
                except Data.DoesNotExist:
                    logger.warning(
                        "Data object does not exist (handle_finish).",
                        extra={
                            'data_id': data_id,
                        }
                    )
                    async_to_sync(self._send_reply)(obj, {ExecutorProtocol.RESULT: ExecutorProtocol.RESULT_ERROR})
                    return

                changeset = {
                    'process_progress': 100,
                    'finished': now(),
                }

                if spawning_failed:
                    changeset['status'] = Data.STATUS_ERROR
                    changeset['process_error'] = ["Error while preparing spawned Data objects"]

                elif process_rc == 0 and not d.status == Data.STATUS_ERROR:
                    changeset['status'] = Data.STATUS_DONE

                else:
                    changeset['status'] = Data.STATUS_ERROR
                    changeset['process_rc'] = process_rc

                obj[ExecutorProtocol.UPDATE_CHANGESET] = changeset
                self.handle_update(obj, internal_call=True)

        if not getattr(settings, 'FLOW_MANAGER_KEEP_DATA', False):
            # Purge worker is not running in test runner, so we should skip triggering it.
            if not is_testing():
                channel_layer = get_channel_layer()
                try:
                    async_to_sync(channel_layer.send)(
                        CHANNEL_PURGE_WORKER,
                        {
                            'type': TYPE_PURGE_RUN,
                            'location_id': d.location.id,
                            'verbosity': self._verbosity,
                        }
                    )
                except ChannelFull:
                    logger.warning(
                        "Cannot trigger purge because channel is full.",
                        extra={'data_id': data_id}
                    )

        # Notify the executor that we're done.
        async_to_sync(self._send_reply)(obj, {ExecutorProtocol.RESULT: ExecutorProtocol.RESULT_OK})

        # Now nudge the main manager to perform final cleanup. This is
        # needed even if there was no spawn baggage, since the manager
        # may need to know when executors have finished, to keep count
        # of them and manage synchronization.
        async_to_sync(consumer.send_event)({
            WorkerProtocol.COMMAND: WorkerProtocol.FINISH,
            WorkerProtocol.DATA_ID: data_id,
            WorkerProtocol.FINISH_SPAWNED: spawned,
            WorkerProtocol.FINISH_COMMUNICATE_EXTRA: {
                'executor': getattr(settings, 'FLOW_EXECUTOR', {}).get('NAME', 'resolwe.flow.executors.local'),
            },
        })


Instruction:
Can you tell what is the following Python 3 function doing
def handle_abort(self, obj):
        async_to_sync(consumer.send_event)({
            WorkerProtocol.COMMAND: WorkerProtocol.ABORT,
            WorkerProtocol.DATA_ID: obj[ExecutorProtocol.DATA_ID],
            WorkerProtocol.FINISH_COMMUNICATE_EXTRA: {
                'executor': getattr(settings, 'FLOW_EXECUTOR', {}).get('NAME', 'resolwe.flow.executors.local'),
            },
        })

Response: Handle an incoming Abort processing request.


Instruction:
Can you write a function in Python 3 where it
handles an incoming log processing request.

Response:
def handle_log(self, obj):
        """Handle an incoming log processing request.

        :param obj: The Channels message object. Command object format:

            .. code-block:: none

                {
                    'command': 'log',
                    'message': [log message]
                }
        """
        record_dict = json.loads(obj[ExecutorProtocol.LOG_MESSAGE])
        record_dict['msg'] = record_dict['msg']

        executors_dir = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'executors')
        record_dict['pathname'] = os.path.join(executors_dir, record_dict['pathname'])

        logger.handle(logging.makeLogRecord(record_dict))


Instruction:
Write a Python 3 script for
pushing current stats to Redis.

Response:
async def push_stats(self):
        """Push current stats to Redis."""
        snapshot = self._make_stats()
        try:
            serialized = json.dumps(snapshot)
            await self._call_redis(aioredis.Redis.set, state.MANAGER_LISTENER_STATS, serialized)
            await self._call_redis(aioredis.Redis.expire, state.MANAGER_LISTENER_STATS, 3600)
        except TypeError:
            logger.error(__(
                "Listener can't serialize statistics:\n\n{}",
                traceback.format_exc()
            ))
        except aioredis.RedisError:
            logger.error(__(
                "Listener can't store updated statistics:\n\n{}",
                traceback.format_exc()
            ))


Instruction:
How would you implement a function in Python 3 that
checks for critical load and log an error if necessary.

Response:
def check_critical_load(self):
        """Check for critical load and log an error if necessary."""
        if self.load_avg.intervals['1m'].value > 1:
            if self.last_load_level == 1 and time.time() - self.last_load_log < 30:
                return
            self.last_load_log = time.time()
            self.last_load_level = 1
            logger.error(
                "Listener load limit exceeded, the system can't handle this!",
                extra=self._make_stats()
            )

        elif self.load_avg.intervals['1m'].value > 0.8:
            if self.last_load_level == 0.8 and time.time() - self.last_load_log < 30:
                return
            self.last_load_log = time.time()
            self.last_load_level = 0.8
            logger.warning(
                "Listener load approaching critical!",
                extra=self._make_stats()
            )

        else:
            self.last_load_log = -math.inf
            self.last_load_level = 0


Instruction:
Implement a function in Python 3 to
run the main listener run loop. Doesn't return until :meth:`terminate` is called.

Response:
async def run(self):
        """Run the main listener run loop.

        Doesn't return until :meth:`terminate` is called.
        """
        logger.info(__(
            "Starting Resolwe listener on channel '{}'.",
            state.MANAGER_EXECUTOR_CHANNELS.queue
        ))
        while not self._should_stop:
            await self.push_stats()
            ret = await self._call_redis(aioredis.Redis.blpop, state.MANAGER_EXECUTOR_CHANNELS.queue, timeout=1)
            if ret is None:
                self.load_avg.add(0)
                continue
            remaining = await self._call_redis(aioredis.Redis.llen, state.MANAGER_EXECUTOR_CHANNELS.queue)
            self.load_avg.add(remaining + 1)
            self.check_critical_load()
            _, item = ret
            try:
                item = item.decode('utf-8')
                logger.debug(__("Got command from executor: {}", item))
                obj = json.loads(item)
            except json.JSONDecodeError:
                logger.error(
                    __("Undecodable command packet:\n\n{}"),
                    traceback.format_exc()
                )
                continue

            command = obj.get(ExecutorProtocol.COMMAND, None)
            if command is None:
                continue

            service_start = time.perf_counter()

            handler = getattr(self, 'handle_' + command, None)
            if handler:
                try:
                    with PrioritizedBatcher.global_instance():
                        await database_sync_to_async(handler)(obj)
                except Exception:  # pylint: disable=broad-except
                    logger.error(__(
                        "Executor command handling error:\n\n{}",
                        traceback.format_exc()
                    ))
            else:
                logger.error(
                    __("Unknown executor command '{}'.", command),
                    extra={'decoded_packet': obj}
                )

            # We do want to measure wall-clock time elapsed, because
            # system load will impact event handling performance. On
            # a lagging system, good internal performance is meaningless.
            service_end = time.perf_counter()
            self.service_time.update(service_end - service_start)
        logger.info(__(
            "Stopping Resolwe listener on channel '{}'.",
            state.MANAGER_EXECUTOR_CHANNELS.queue
        ))


Instruction:
Can you generate the documentation for the following Python 3 function
def dokanMain(self, dokanOptions, dokanOperations):
        return int(
            self.dokanDLL.DokanMain(
                PDOKAN_OPTIONS(dokanOptions), PDOKAN_OPERATIONS(dokanOperations)
            )
        )

Response: Issue callback to start dokan drive.


Instruction:
Explain what the following Python 3 code does
def createFile(
        self,
        fileName,
        desiredAccess,
        shareMode,
        creationDisposition,
        flagsAndAttributes,
        dokanFileInfo,
    ):
        """Creates a file.

        :param fileName: name of file to create
        :type fileName: ctypes.c_wchar_p
        :param desiredAccess: desired access flags
        :type desiredAccess: ctypes.c_ulong
        :param shareMode: share mode flags
        :type shareMode: ctypes.c_ulong
        :param creationDisposition: creation disposition flags
        :type creationDisposition: ctypes.c_ulong
        :param flagsAndAttributes: creation flags and attributes
        :type flagsAndAttributes: ctypes.c_ulong
        :param dokanFileInfo: used by Dokan
        :type dokanFileInfo: PDOKAN_FILE_INFO
        :return: error code
        :rtype: ctypes.c_int

        """
        return self.operations('createFile', fileName)

Response: Creates a file in the cache.


Instruction:
Implement a Python 3 function for
reading a file from the Dokan and returns the size of the file.

Response:
def readFile(
        self,
        fileName,
        buffer,
        numberOfBytesToRead,
        numberOfBytesRead,
        offset,
        dokanFileInfo,
    ):
        """Read a file.

        :param fileName: name of file to read
        :type fileName: ctypes.c_wchar_p
        :param buffer: buffer for content read
        :type buffer: ctypes.c_void_p
        :param numberOfBytesToRead: number of bytes to read
        :type numberOfBytesToRead: ctypes.c_ulong
        :param numberOfBytesRead: number of bytes read
        :type numberOfBytesRead: ctypes.POINTER(ctypes.c_ulong)
        :param offset: byte offset
        :type offset: ctypes.c_longlong
        :param dokanFileInfo: used by Dokan
        :type dokanFileInfo: PDOKAN_FILE_INFO
        :return: error code
        :rtype: ctypes.c_int

        """
        try:
            ret = self.operations('readFile', fileName, numberOfBytesToRead, offset)
            data = ctypes.create_string_buffer(
                ret[:numberOfBytesToRead], numberOfBytesToRead
            )
            ctypes.memmove(buffer, data, numberOfBytesToRead)
            sizeRead = ctypes.c_ulong(len(ret))
            ctypes.memmove(
                numberOfBytesRead, ctypes.byref(sizeRead), ctypes.sizeof(ctypes.c_ulong)
            )
            return d1_onedrive.impl.drivers.dokan.const.DOKAN_SUCCESS
        except Exception:
            # logging.error('%s', e)
            return d1_onedrive.impl.drivers.dokan.const.DOKAN_ERROR


Instruction:
Can you implement a function in Python 3 that
writes a file from the cache.

Response:
def writeFile(
        self,
        fileName,
        buffer,
        numberOfBytesToWrite,
        numberOfBytesWritten,
        offset,
        dokanFileInfo,
    ):
        """Read a file.

        :param fileName: name of file to write
        :type fileName: ctypes.c_wchar_p
        :param buffer: buffer to write
        :type buffer: ctypes.c_void_p
        :param numberOfBytesToWrite: number of bytes to write
        :type numberOfBytesToWrite: ctypes.c_ulong
        :param numberOfBytesWritten: number of bytes written
        :type numberOfBytesWritten: ctypes.POINTER(ctypes.c_ulong)
        :param offset: byte offset
        :type offset: ctypes.c_longlong
        :param dokanFileInfo: used by Dokan
        :type dokanFileInfo: PDOKAN_FILE_INFO
        :return: error code
        :rtype: ctypes.c_int

        """
        return self.operations(
            'writeFile', fileName, buffer, numberOfBytesToWrite, offset
        )


Instruction:
Create a Python 3 function for
finding files in a certain path that match the search pattern. :param fileName: path to search :type fileName: ctypes.c_wchar_p :param searchPattern: pattern to search for :type searchPattern: ctypes.c_wchar_p :param fillFindData: function pointer for populating search results :type fillFindData: PFillFindData :param dokanFileInfo: used by Dokan :type dokanFileInfo: PDOKAN_FILE_INFO :return: error code :rtype: ctypes.c_int

Response:
def findFilesWithPattern(
        self, fileName, searchPattern, fillFindData, dokanFileInfo
    ):
        """Find files in a certain path that match the search pattern.

        :param fileName: path to search
        :type fileName: ctypes.c_wchar_p
        :param searchPattern: pattern to search for
        :type searchPattern: ctypes.c_wchar_p
        :param fillFindData: function pointer for populating search results
        :type fillFindData: PFillFindData
        :param dokanFileInfo: used by Dokan
        :type dokanFileInfo: PDOKAN_FILE_INFO
        :return: error code
        :rtype: ctypes.c_int

        """
        try:
            ret = self.operations('findFilesWithPattern', fileName, searchPattern)
            if ret is None:
                return d1_onedrive.impl.drivers.dokan.const.DOKAN_ERROR
            for r in ret:
                create_ft = self.python_timestamp_to_win32_filetime(r['ctime'])
                last_access_ft = self.python_timestamp_to_win32_filetime(r['atime'])
                last_write_ft = self.python_timestamp_to_win32_filetime(r['wtime'])
                cft = ctypes.wintypes.FILETIME(create_ft[0], create_ft[1])
                laft = ctypes.wintypes.FILETIME(last_access_ft[0], last_access_ft[1])
                lwft = ctypes.wintypes.FILETIME(last_write_ft[0], last_write_ft[1])
                size = self.pyint_to_double_dwords(r['size'])
                File = ctypes.wintypes.WIN32_FIND_DATAW(
                    ctypes.c_ulong(r['attr']),  # attributes
                    cft,  # creation time
                    laft,  # last access time
                    lwft,  # last write time
                    size[1],  # upper bits of size
                    size[0],  # lower bits of size
                    ctypes.c_ulong(0),  # reserved for FS
                    ctypes.c_ulong(0),  # reserved for FS
                    r['name'],  # file name
                    '',
                )  # alternate name
                pFile = ctypes.wintypes.PWIN32_FIND_DATAW(File)
                fillFindData(pFile, dokanFileInfo)
            return d1_onedrive.impl.drivers.dokan.const.DOKAN_SUCCESS
        except Exception as e:
            logging.error('%s', e)
            return d1_onedrive.impl.drivers.dokan.const.DOKAN_ERROR


Instruction:
Here you have a function in Python 3, explain what it does
def setFileTime(
        self, fileName, creationTime, lastAccessTime, lastWriteTime, dokanFileInfo
    ):
        """Set time values for a file.

        :param fileName: name of file to set time values for
        :type fileName: ctypes.c_wchar_p
        :param creationTime: creation time of file
        :type creationTime: ctypes.POINTER(ctypes.wintypes.FILETIME)
        :param lastAccessTime: last access time of file
        :type lastAccessTime: ctypes.POINTER(ctypes.wintypes.FILETIME)
        :param lastWriteTime: last write time of file
        :type lastWriteTime: ctypes.POINTER(ctypes.wintypes.FILETIME)
        :param dokanFileInfo: used by Dokan
        :type dokanFileInfo: PDOKAN_FILE_INFO
        :return: error code
        :rtype: ctypes.c_int

        """
        return self.operations('setFileTime', fileName)

Response: Set the file time values for a specific file.


Instruction:
Explain what the following Python 3 code does
def moveFile(self, existingFileName, newFileName, replaceExisiting, dokanFileInfo):
        return self.operations('moveFile', existingFileName, newFileName)

Response:
Move a file.

        :param existingFileName: name of file to move
        :type existingFileName: ctypes.c_wchar_p
        :param newFileName: new name of file
        :type newFileName: ctypes.c_wchar_p
        :param replaceExisting: flag to indicate replacement of existing file
        :type replaceExisting: ctypes.c_bool
        :param dokanFileInfo: used by Dokan
        :type dokanFileInfo: PDOKAN_FILE_INFO
        :return: error code
        :rtype: ctypes.c_int


Instruction:
Explain what the following Python 3 code does
def lockFile(self, fileName, byteOffset, length, dokanFileInfo):
        return self.operations('lockFile', fileName, byteOffset, length)

Response:
Lock a file.

        :param fileName: name of file to lock
        :type fileName: ctypes.c_wchar_p
        :param byteOffset: location to start lock
        :type byteOffset: ctypes.c_longlong
        :param length: number of bytes to lock
        :type length: ctypes.c_longlong
        :param dokanFileInfo: used by Dokan
        :type dokanFileInfo: PDOKAN_FILE_INFO
        :return: error code
        :rtype: ctypes.c_int


Instruction:
How would you implement a function in Python 3 that
unlocks a file. :param fileName: name of file to unlock :type fileName: ctypes.c_wchar_p :param byteOffset: location to start unlock :type byteOffset: ctypes.c_longlong :param length: number of bytes to unlock :type length: ctypes.c_longlong :param dokanFileInfo: used by Dokan :type dokanFileInfo: PDOKAN_FILE_INFO :return: error code :rtype: ctypes.c_int

Response:
def unlockFile(self, fileName, byteOffset, length, dokanFileInfo):
        """Unlock a file.

        :param fileName: name of file to unlock
        :type fileName: ctypes.c_wchar_p
        :param byteOffset: location to start unlock
        :type byteOffset: ctypes.c_longlong
        :param length: number of bytes to unlock
        :type length: ctypes.c_longlong
        :param dokanFileInfo: used by Dokan
        :type dokanFileInfo: PDOKAN_FILE_INFO
        :return: error code
        :rtype: ctypes.c_int

        """
        return self.operations('unlockFile', fileName, byteOffset, length)


Instruction:
Make a summary of the following Python 3 code
def getDiskFreeSpace(
        self,
        freeBytesAvailable,
        totalNumberOfBytes,
        totalNumberOfFreeBytes,
        dokanFileInfo,
    ):
        """Get the amount of free space on this volume.

        :param freeBytesAvailable: pointer for free bytes available
        :type freeBytesAvailable: ctypes.c_void_p
        :param totalNumberOfBytes: pointer for total number of bytes
        :type totalNumberOfBytes: ctypes.c_void_p
        :param totalNumberOfFreeBytes: pointer for total number of free bytes
        :type totalNumberOfFreeBytes: ctypes.c_void_p
        :param dokanFileInfo: used by Dokan
        :type dokanFileInfo: PDOKAN_FILE_INFO
        :return: error code
        :rtype: ctypes.c_int

        """
        ret = self.operations('getDiskFreeSpace')
        ctypes.memmove(
            freeBytesAvailable,
            ctypes.byref(ctypes.c_longlong(ret['freeBytesAvailable'])),
            ctypes.sizeof(ctypes.c_longlong),
        )
        ctypes.memmove(
            totalNumberOfBytes,
            ctypes.byref(ctypes.c_longlong(ret['totalNumberOfBytes'])),
            ctypes.sizeof(ctypes.c_longlong),
        )
        ctypes.memmove(
            totalNumberOfFreeBytes,
            ctypes.byref(ctypes.c_longlong(ret['totalNumberOfFreeBytes'])),
            ctypes.sizeof(ctypes.c_longlong),
        )
        return d1_onedrive.impl.drivers.dokan.const.DOKAN_SUCCESS

Response: This method returns the amount of free space on this volume.


Instruction:
Create a Python 3 function to
get information about the volume. :param volumeNameBuffer: buffer for volume name :type volumeNameBuffer: ctypes.c_void_p :param volumeNameSize: volume name buffer size :type volumeNameSize: ctypes.c_ulong :param volumeSerialNumber: buffer for volume serial number :type volumeSerialNumber: ctypes.c_void_p :param maximumComponentLength: buffer for maximum component length :type maximumComponentLength: ctypes.c_void_p :param fileSystemFlags: buffer for file system flags :type fileSystemFlags: ctypes.c_void_p :param fileSystemNameBuffer: buffer for file system name :type fileSystemNameBuffer: ctypes.c_void_p :param fileSystemNameSize: file system name buffer size :type fileSystemNameSize: ctypes.c_ulong :param dokanFileInfo: used by Dokan :type dokanFileInfo: PDOKAN_FILE_INFO :return: error code :rtype: ctypes.c_int

Response:
def getVolumeInformation(
        self,
        volumeNameBuffer,
        volumeNameSize,
        volumeSerialNumber,
        maximumComponentLength,
        fileSystemFlags,
        fileSystemNameBuffer,
        fileSystemNameSize,
        dokanFileInfo,
    ):
        """Get information about the volume.

        :param volumeNameBuffer: buffer for volume name
        :type volumeNameBuffer: ctypes.c_void_p
        :param volumeNameSize: volume name buffer size
        :type volumeNameSize: ctypes.c_ulong
        :param volumeSerialNumber: buffer for volume serial number
        :type volumeSerialNumber: ctypes.c_void_p
        :param maximumComponentLength: buffer for maximum component length
        :type maximumComponentLength: ctypes.c_void_p
        :param fileSystemFlags: buffer for file system flags
        :type fileSystemFlags: ctypes.c_void_p
        :param fileSystemNameBuffer: buffer for file system name
        :type fileSystemNameBuffer: ctypes.c_void_p
        :param fileSystemNameSize: file system name buffer size
        :type fileSystemNameSize: ctypes.c_ulong
        :param dokanFileInfo: used by Dokan
        :type dokanFileInfo: PDOKAN_FILE_INFO
        :return: error code
        :rtype: ctypes.c_int

        """
        ret = self.operations('getVolumeInformation')
        # populate volume name buffer
        ctypes.memmove(
            volumeNameBuffer,
            ret['volumeNameBuffer'],
            min(
                ctypes.sizeof(ctypes.c_wchar) * len(ret['volumeNameBuffer']),
                volumeNameSize,
            ),
        )
        # populate serial number buffer
        serialNum = ctypes.c_ulong(self.serialNumber)
        ctypes.memmove(
            volumeSerialNumber, ctypes.byref(serialNum), ctypes.sizeof(ctypes.c_ulong)
        )
        # populate max component length
        maxCompLen = ctypes.c_ulong(ret['maximumComponentLength'])
        ctypes.memmove(
            maximumComponentLength,
            ctypes.byref(maxCompLen),
            ctypes.sizeof(ctypes.c_ulong),
        )
        # populate filesystem flags buffer
        fsFlags = ctypes.c_ulong(ret['fileSystemFlags'])
        ctypes.memmove(
            fileSystemFlags, ctypes.byref(fsFlags), ctypes.sizeof(ctypes.c_ulong)
        )
        # populate filesystem name
        ctypes.memmove(
            fileSystemNameBuffer,
            ret['fileSystemNameBuffer'],
            min(
                ctypes.sizeof(ctypes.c_wchar) * len(ret['fileSystemNameBuffer']),
                fileSystemNameSize,
            ),
        )
        return d1_onedrive.impl.drivers.dokan.const.DOKAN_SUCCESS


Instruction:
Can you tell what is the following Python 3 function doing
def getFileSecurity(
        self,
        fileName,
        securityInformation,
        securityDescriptor,
        lengthSecurityDescriptorBuffer,
        lengthNeeded,
        dokanFileInfo,
    ):
        """Get security attributes of a file.

        :param fileName: name of file to get security for
        :type fileName: ctypes.c_wchar_p
        :param securityInformation: buffer for security information
        :type securityInformation: PSECURITY_INFORMATION
        :param securityDescriptor: buffer for security descriptor
        :type securityDescriptor: PSECURITY_DESCRIPTOR
        :param lengthSecurityDescriptorBuffer: length of descriptor buffer
        :type lengthSecurityDescriptorBuffer: ctypes.c_ulong
        :param lengthNeeded: length needed for the buffer
        :type lengthNeeded: ctypes.POINTER(ctypes.c_ulong)
        :param dokanFileInfo: used by Dokan
        :type dokanFileInfo: PDOKAN_FILE_INFO
        :return: error code
        :rtype: ctypes.c_int

        """
        return self.operations('getFileSecurity', fileName)

Response: Get security attributes of a file.


Instruction:
Write a Python 3 function that can
set security attributes of a file.

Response:
def setFileSecurity(
        self,
        fileName,
        securityInformation,
        securityDescriptor,
        lengthSecurityDescriptorBuffer,
        dokanFileInfo,
    ):
        """Set security attributes of a file.

        :param fileName: name of file to set security for
        :type fileName: ctypes.c_wchar_p
        :param securityInformation: new security information
        :type securityInformation: PSECURITY_INFORMATION
        :param securityDescriptor: newsecurity descriptor
        :type securityDescriptor: PSECURITY_DESCRIPTOR
        :param lengthSecurityDescriptorBuffer: length of descriptor buffer
        :type lengthSecurityDescriptorBuffer: ctypes.c_ulong
        :param dokanFileInfo: used by Dokan
        :type dokanFileInfo: PDOKAN_FILE_INFO
        :return: error code
        :rtype: ctypes.c_int

        """
        return self.operations('setFileSecurity', fileName)


Instruction:
Can you create a Python 3 function that
creates a simple resource map with one metadata document and n data objects.

Response:
def createSimpleResourceMap(ore_pid, sci_meta_pid, data_pids):
    """Create a simple resource map with one metadata document and n data objects."""
    ore = ResourceMap()
    ore.initialize(ore_pid)
    ore.addMetadataDocument(sci_meta_pid)
    ore.addDataDocuments(data_pids, sci_meta_pid)
    return ore


Instruction:
Write a Python 3 function for
reading pids from in_stream and generate a resource map.

Response:
def pids2ore(in_stream, fmt='xml', base_url='https://cn.dataone.org/cn'):
    """read pids from in_stream and generate a resource map.

    first pid is the ore_pid second is the sci meta pid remainder are data pids

    """
    pids = []
    for line in in_stream:
        pid = line.strip()
        if len(pid) > 0:
            if not pid.startswith("# "):
                pids.append(pid)
    if (len(pids)) < 2:
        raise ValueError("Insufficient identifiers provided.")

    logging.info("Read %d identifiers", len(pids))

    ore = ResourceMap(base_url=base_url)

    logging.info("ORE PID = %s", pids[0])
    ore.initialize(pids[0])

    logging.info("Metadata PID = %s", pids[1])
    ore.addMetadataDocument(pids[1])

    ore.addDataDocuments(pids[2:], pids[1])
    return ore.serialize_to_display(doc_format=fmt)


Instruction:
Make a summary of the following Python 3 code
def submit(self, data, runtime_dir, argv):
        queue = 'ordinary'
        if data.process.scheduling_class == Process.SCHEDULING_CLASS_INTERACTIVE:
            queue = 'hipri'

        logger.debug(__(
            "Connector '{}' running for Data with id {} ({}) in celery queue {}, EAGER is {}.",
            self.__class__.__module__,
            data.id,
            repr(argv),
            queue,
            getattr(settings, 'CELERY_ALWAYS_EAGER', None)
        ))
        celery_run.apply_async((data.id, runtime_dir, argv), queue=queue)

Response: Submit a new process.


Instruction:
Create a Python 3 function to
synchronize the local tree of Solr records for DataONE identifiers and queries with the reference tree.

Response:
def refresh(self):
        """Synchronize the local tree of Solr records for DataONE identifiers and
        queries with the reference tree."""
        if self._source_tree.cache_is_stale():
            self._source_tree.refresh()
            logging.info('Refreshing object tree')
            self._init_cache()
            self.sync_cache_with_source_tree()


Instruction:
How would you explain what the following Python 3 function does
def get_object_record(self, pid):
        try:
            return self._cache['records'][pid]
        except KeyError:
            raise d1_onedrive.impl.onedrive_exceptions.ONEDriveException('Unknown PID')

Response: Get an object record from the object tree.


Instruction:
Given the following Python 3 function, write the documentation
def get_object_record_with_sync(self, pid):
        try:
            return self._cache['records'][pid]
        except KeyError:
            return self._get_uncached_object_record(pid)

Response: Get an object record with sync.


Instruction:
Can you write a function in Python 3 where it
creates a cache item for a given PID.

Response:
def _create_cache_item_for_pid(self, cache_folder, pid):
        """The source tree can contain identifiers that are no longer valid (or were
        never valid).

        Any items for which a Solr record cannot be retrieved are silently skipped.

        """
        try:
            record = self._solr_client.get_solr_record(pid)
        except d1_onedrive.impl.onedrive_exceptions.ONEDriveException:
            pass
        else:
            self._create_cache_item(cache_folder, record)


Instruction:
How would you code a function in Python 3 to
return abstracted status of dependencies.

Response:
def dependency_status(data):
    """Return abstracted status of dependencies.

    - ``STATUS_ERROR`` .. one dependency has error status or was deleted
    - ``STATUS_DONE`` .. all dependencies have done status
    - ``None`` .. other

    """
    parents_statuses = set(
        DataDependency.objects.filter(
            child=data, kind=DataDependency.KIND_IO
        ).distinct('parent__status').values_list('parent__status', flat=True)
    )

    if not parents_statuses:
        return Data.STATUS_DONE

    if None in parents_statuses:
        # Some parents have been deleted.
        return Data.STATUS_ERROR

    if Data.STATUS_ERROR in parents_statuses:
        return Data.STATUS_ERROR

    if len(parents_statuses) == 1 and Data.STATUS_DONE in parents_statuses:
        return Data.STATUS_DONE

    return None


Instruction:
Can you generate the documentation for the following Python 3 function
def discover_engines(self, executor=None):
        if executor is None:
            executor = getattr(settings, 'FLOW_EXECUTOR', {}).get('NAME', 'resolwe.flow.executors.local')
        self.executor = self.load_executor(executor)
        logger.info(
            __("Loaded '{}' executor.", str(self.executor.__class__.__module__).replace('.prepare', ''))
        )

        expression_engines = getattr(settings, 'FLOW_EXPRESSION_ENGINES', ['resolwe.flow.expression_engines.jinja'])
        self.expression_engines = self.load_expression_engines(expression_engines)
        logger.info(__(
            "Found {} expression engines: {}", len(self.expression_engines), ', '.join(self.expression_engines.keys())
        ))

        execution_engines = getattr(settings, 'FLOW_EXECUTION_ENGINES', ['resolwe.flow.execution_engines.bash'])
        self.execution_engines = self.load_execution_engines(execution_engines)
        logger.info(__(
            "Found {} execution engines: {}", len(self.execution_engines), ', '.join(self.execution_engines.keys())
        ))

Response:
Discover configured engines.

        :param executor: Optional executor module override


Instruction:
Can you tell what is the following Python 3 function doing
def reset(self, keep_state=False):
        if not keep_state:
            self.state = state.ManagerState(state.MANAGER_STATE_PREFIX)
            self.state.reset()
        async_to_sync(consumer.run_consumer)(timeout=1)
        async_to_sync(self.sync_counter.reset)()

Response: Reset the shared state and drain Django Channels.


Instruction:
Implement a Python 3 function for
marshaling Django settings into a serializable object.

Response:
def _marshal_settings(self):
        """Marshal Django settings into a serializable object.

        :return: The serialized settings.
        :rtype: dict
        """
        result = {}
        for key in dir(settings):
            if any(map(key.startswith, ['FLOW_', 'RESOLWE_', 'CELERY_'])):
                result[key] = getattr(settings, key)
        return result


Instruction:
Implement a function in Python 3 to
select a concrete connector and run the process through it.

Response:
def run(self, data, runtime_dir, argv):
        """Select a concrete connector and run the process through it.

        :param data: The :class:`~resolwe.flow.models.Data` object that
            is to be run.
        :param runtime_dir: The directory the executor is run from.
        :param argv: The argument vector used to spawn the executor.
        """
        process_scheduling = self.scheduling_class_map[data.process.scheduling_class]
        if 'DISPATCHER_MAPPING' in getattr(settings, 'FLOW_MANAGER', {}):
            class_name = settings.FLOW_MANAGER['DISPATCHER_MAPPING'][process_scheduling]
        else:
            class_name = getattr(settings, 'FLOW_MANAGER', {}).get('NAME', DEFAULT_CONNECTOR)

        data.scheduled = now()
        data.save(update_fields=['scheduled'])

        async_to_sync(self.sync_counter.inc)('executor')
        return self.connectors[class_name].submit(data, runtime_dir, argv)


Instruction:
Can you tell what is the following Python 3 function doing
def _get_per_data_dir(self, dir_base, subpath):
        # Use Django settings here, because the state must be preserved
        # across events. This also implies the directory settings can't
        # be patched outside the manager and then just sent along in the
        # command packets.
        result = self.settings_actual.get('FLOW_EXECUTOR', {}).get(dir_base, '')
        return os.path.join(result, subpath)

Response: Extend the given base directory with a per - data component.