Instruction:
How would you code a function in Python 3 to
handle the list_docker_images command.

Response:
def handle(self, *args, **options):
        """Handle command list_docker_images."""
        verbosity = int(options.get('verbosity'))

        # Check that the specified output format is valid
        if options['format'] != 'plain' and options['format'] != 'yaml':
            raise CommandError("Unknown output format: %s" % options['format'])

        # Gather only unique latest custom Docker requirements that the processes are using
        # The 'image' field is optional, so be careful about that as well
        unique_docker_images = set(
            p.requirements['executor']['docker']['image']
            for p in Process.objects.filter(is_active=True).order_by(
                'slug', '-version'
            ).distinct(
                'slug'
            ).only(
                'requirements'
            ).filter(
                requirements__icontains='docker'
            )
            if 'image' in p.requirements.get('executor', {}).get('docker', {})
        )

        # Add the default image.
        unique_docker_images.add(DEFAULT_CONTAINER_IMAGE)

        # Pull images if requested or just output the list in specified format
        if options['pull']:
            # Remove set of already pulled images.
            with PULLED_IMAGES_LOCK:
                unique_docker_images.difference_update(PULLED_IMAGES)

            # Get the desired 'docker' command from settings or use the default
            docker = getattr(settings, 'FLOW_DOCKER_COMMAND', 'docker')

            # Pull each image
            for img in unique_docker_images:
                ret = subprocess.call(
                    shlex.split('{} pull {}'.format(docker, img)),
                    stdout=None if verbosity > 0 else subprocess.DEVNULL,
                    stderr=None if verbosity > 0 else subprocess.DEVNULL,
                )

                # Update set of pulled images.
                with PULLED_IMAGES_LOCK:
                    PULLED_IMAGES.add(img)

                if ret != 0:
                    errmsg = "Failed to pull Docker image '{}'!".format(img)

                    if not options['ignore_pull_errors']:
                        # Print error and stop execution
                        raise CommandError(errmsg)
                    else:
                        # Print error, but keep going
                        logger.error(errmsg)
                        if verbosity > 0:
                            self.stderr.write(errmsg)
                else:
                    msg = "Docker image '{}' pulled successfully!".format(img)
                    logger.info(msg)
                    if verbosity > 0:
                        self.stdout.write(msg)
        else:
            # Sort the set of unique Docker images for nicer output.
            unique_docker_images = sorted(unique_docker_images)

            # Convert the set of unique Docker images into a list of dicts for easier output
            imgs = [
                dict(name=s[0], tag=s[1] if len(s) == 2 else 'latest')
                for s in (img.split(':') for img in unique_docker_images)
            ]

            # Output in YAML or plaintext (one image per line), as requested
            if options['format'] == 'yaml':
                out = yaml.safe_dump(imgs, default_flow_style=True, default_style="'")
            else:
                out = functools.reduce(operator.add,
                                       ('{name}:{tag}\n'.format(**i) for i in imgs), '')

            self.stdout.write(out, ending='')


Instruction:
Can you implement a function in Python 3 that
gets or create a Data object if similar already exists otherwise create it.

Response:
def get_or_create(self, request, *args, **kwargs):
        """Get ``Data`` object if similar already exists, otherwise create it."""
        kwargs['get_or_create'] = True
        return self.create(request, *args, **kwargs)


Instruction:
Create a Python 3 function to
perform get_or_create - return existing object if found.

Response:
def perform_get_or_create(self, request, *args, **kwargs):
        """Perform "get_or_create" - return existing object if found."""
        serializer = self.get_serializer(data=request.data)
        serializer.is_valid(raise_exception=True)
        process = serializer.validated_data.get('process')
        process_input = request.data.get('input', {})

        fill_with_defaults(process_input, process.input_schema)

        checksum = get_data_checksum(process_input, process.slug, process.version)
        data_qs = Data.objects.filter(
            checksum=checksum,
            process__persistence__in=[Process.PERSISTENCE_CACHED, Process.PERSISTENCE_TEMP],
        )
        data_qs = get_objects_for_user(request.user, 'view_data', data_qs)
        if data_qs.exists():
            data = data_qs.order_by('created').last()
            serializer = self.get_serializer(data)
            return Response(serializer.data)


Instruction:
Explain what the following Python 3 code does
def ready(self):
        # Stop the startup code from running automatically from pytest unit tests.
        # When running tests in parallel with xdist, an instance of GMN is launched
        # before thread specific settings have been applied.
        # if hasattr(sys, '_launched_by_pytest'):
        #   return
        self._assert_readable_file_if_set('CLIENT_CERT_PATH')
        self._assert_readable_file_if_set('CLIENT_CERT_PRIVATE_KEY_PATH')
        self._assert_dirs_exist('OBJECT_FORMAT_CACHE_PATH')

        self._assert_is_type('SCIMETA_VALIDATION_ENABLED', bool)
        self._assert_is_type('SCIMETA_VALIDATION_MAX_SIZE', int)
        self._assert_is_in('SCIMETA_VALIDATION_OVER_SIZE_ACTION', ('reject', 'accept'))

        self._warn_unsafe_for_prod()
        self._check_resource_map_create()
        if not d1_gmn.app.sciobj_store.is_existing_store():
            self._create_sciobj_store_root()

        self._add_xslt_mimetype()

Response: Called once per Django process instance.


Instruction:
Write a Python 3 function for
checking that the dirs leading up to the given file path exist.

Response:
def _assert_dirs_exist(self, setting_name):
        """Check that the dirs leading up to the given file path exist.

        Does not check if the file exists.

        """
        v = self._get_setting(setting_name)
        if (not os.path.isdir(os.path.split(v)[0])) or os.path.isdir(v):
            self.raise_config_error(
                setting_name,
                v,
                str,
                'a file path in an existing directory',
                is_none_allowed=False,
            )


Instruction:
Here you have a function in Python 3, explain what it does
def _warn_unsafe_for_prod(self):
        safe_settings_list = [
            ('DEBUG', False),
            ('DEBUG_GMN', False),
            ('STAND_ALONE', False),
            ('DATABASES.default.ATOMIC_REQUESTS', True),
            ('SECRET_KEY', '<Do not modify this placeholder value>'),
            ('STATIC_SERVER', False),
        ]
        for setting_str, setting_safe in safe_settings_list:
            setting_current = self._get_setting(setting_str)
            if setting_current != setting_safe:
                logger.warning(
                    'Setting is unsafe for use in production. setting="{}" current="{}" '
                    'safe="{}"'.format(setting_str, setting_current, setting_safe)
                )

Response: Warn on settings that are not safe for production.


Instruction:
Create a Python 3 function to
get a potentially nested dict setting.

Response:
def _get_setting(self, setting_dotted_name, default=None):
        """Return the value of a potentially nested dict setting.

        E.g., 'DATABASES.default.NAME

        """
        name_list = setting_dotted_name.split('.')
        setting_obj = getattr(django.conf.settings, name_list[0], default)
        # if len(name_list) == 1:
        #   return setting_obj

        return functools.reduce(
            lambda o, a: o.get(a, default), [setting_obj] + name_list[1:]
        )


Instruction:
Write a Python 3 script for
refreshing the connection to Elasticsearch when worker is started.

Response:
def _refresh_connection(self):
        """Refresh connection to Elasticsearch when worker is started.

        File descriptors (sockets) can be shared between multiple
        threads. If same connection is used by multiple threads at the
        same time, this can cause timeouts in some of the pushes. So
        connection needs to be reestablished in each thread to make sure
        that it is unique per thread.
        """
        # Thread with same id can be created when one terminates, but it
        # is ok, as we are only concerned about concurent pushes.
        current_thread_id = threading.current_thread().ident

        if current_thread_id != self.connection_thread_id:
            prepare_connection()

            self.connection_thread_id = current_thread_id


Instruction:
Can you write a function in Python 3 where it
generates unique document id for ElasticSearch.

Response:
def generate_id(self, obj):
        """Generate unique document id for ElasticSearch."""
        object_type = type(obj).__name__.lower()
        return '{}_{}'.format(object_type, self.get_object_id(obj))


Instruction:
Make a summary of the following Python 3 code
def process_object(self, obj):
        document = self.document_class(meta={'id': self.generate_id(obj)})

        for field in document._doc_type.mapping:  # pylint: disable=protected-access
            if field in ['users_with_permissions', 'groups_with_permissions', 'public_permission']:
                continue  # These fields are handled separately

            try:
                # use get_X_value function
                get_value_function = getattr(self, 'get_{}_value'.format(field), None)
                if get_value_function:
                    setattr(document, field, get_value_function(obj))  # pylint: disable=not-callable
                    continue

                # use `mapping` dict
                if field in self.mapping:
                    if callable(self.mapping[field]):
                        setattr(document, field, self.mapping[field](obj))
                        continue

                    try:
                        object_attr = dict_dot(obj, self.mapping[field])
                    except (KeyError, AttributeError):
                        object_attr = None

                    if callable(object_attr):
                        # use method on object
                        setattr(document, field, object_attr(obj))
                    else:
                        # use attribute on object
                        setattr(document, field, object_attr)
                    continue

                # get value from the object
                try:
                    object_value = dict_dot(obj, field)
                    setattr(document, field, object_value)
                    continue
                except KeyError:
                    pass

                raise AttributeError("Cannot determine mapping for field {}".format(field))

            except Exception:  # pylint: disable=broad-except
                logger.exception(
                    "Error occurred while setting value of field '%s' in '%s' Elasticsearch index.",
                    field, self.__class__.__name__,
                    extra={'object_type': self.object_type, 'obj_id': obj.pk}
                )

        permissions = self.get_permissions(obj)
        document.users_with_permissions = permissions['users']
        document.groups_with_permissions = permissions['groups']
        document.public_permission = permissions['public']

        self.push_queue.append(document)

Response: Process the current object and push it to the ElasticSearch index.


Instruction:
Create a Python 3 function to
create the mappings in elasticsearch.

Response:
def create_mapping(self):
        """Create the mappings in elasticsearch."""
        try:
            self.document_class.init()
            self._mapping_created = True
        except IllegalOperation as error:
            if error.args[0].startswith('You cannot update analysis configuration'):
                # Ignore mapping update errors, which are thrown even when the analysis
                # configuration stays the same.
                # TODO: Remove this when https://github.com/elastic/elasticsearch-dsl-py/pull/272 is merged.
                return

            raise


Instruction:
Write a Python 3 function that can
push built documents to Elasticsearch server.

Response:
def push(self):
        """Push built documents to ElasticSearch."""
        self._refresh_connection()

        # Check if we need to update mappings as this needs to be done
        # before we push anything to the Elasticsearch server.
        # This must be done even if the queue is empty, as otherwise ES
        # will fail when retrieving data.
        if not self._mapping_created:
            logger.debug("Pushing mapping for Elasticsearch index '%s'.", self.__class__.__name__)
            self.create_mapping()

        if not self.push_queue:
            logger.debug("No documents to push, skipping push.")
            return

        logger.debug("Found %s documents to push to Elasticsearch.", len(self.push_queue))

        bulk(connections.get_connection(), (doc.to_dict(True) for doc in self.push_queue), refresh=True)
        self.push_queue = []

        logger.debug("Finished pushing builded documents to Elasticsearch server.")


Instruction:
Can you create a Python 3 function that
returns users and groups with view permission on the current object.

Response:
def get_permissions(self, obj):
        """Return users and groups with ``view`` permission on the current object.

        Return a dict with two keys - ``users`` and ``groups`` - which
        contain list of ids of users/groups with ``view`` permission.
        """
        # TODO: Optimize this for bulk running
        filters = {
            'object_pk': obj.id,
            'content_type': ContentType.objects.get_for_model(obj),
            'permission__codename__startswith': 'view',
        }
        return {
            'users': list(
                UserObjectPermission.objects.filter(**filters).distinct('user').values_list('user_id', flat=True)
            ),
            'groups': list(
                GroupObjectPermission.objects.filter(**filters).distinct('group').values_list('group', flat=True)
            ),
            'public': UserObjectPermission.objects.filter(user__username=ANONYMOUS_USER_NAME, **filters).exists(),
        }


Instruction:
Make a summary of the following Python 3 code
def remove_object(self, obj):
        obj_id = self.generate_id(obj)
        es_obj = self.document_class.get(obj_id, ignore=[404])
        # Object may not exist in this index.
        if es_obj:
            es_obj.delete(refresh=True)

Response: Remove current object from the ElasticSearch index.


Instruction:
Can you write a function in Python 3 where it
generates a new entry point for the current locale.

Response:
def generate_pyxb_binding(self, args):
        """Args:

        args:

        """
        pyxbgen_args = []
        pyxbgen_args.append('--schema-root=\'{}\''.format(self.schema_dir))
        pyxbgen_args.append('--binding-root=\'{}\''.format(self.binding_dir))
        pyxbgen_args.append(
            '--schema-stripped-prefix='
            '\'https://repository.dataone.org/software/cicore/branches/D1_SCHEMA_v1.1/\''
        )
        pyxbgen_args.extend(args)

        self.run_pyxbgen(pyxbgen_args)


Instruction:
How would you explain what the following Python 3 function does
def generate_version_file(self, schema_filename, binding_filename):
        version_filename = binding_filename + '_version.txt'
        version_path = os.path.join(self.binding_dir, version_filename)
        schema_path = os.path.join(self.schema_dir, schema_filename)
        try:
            tstamp, svnpath, svnrev, version = self.get_version_info_from_svn(
                schema_path
            )
        except TypeError:
            pass
        else:
            self.write_version_file(version_path, tstamp, svnpath, svnrev, version)

Response:
Given a DataONE schema generates a file that contains version information
            about the schema.


Instruction:
Can you implement a function in Python 3 that
creates DataLocation for each Data.

Response:
def set_data_location(apps, schema_editor):
    """Create DataLocation for each Data."""
    Data = apps.get_model('flow', 'Data')
    DataLocation = apps.get_model('flow', 'DataLocation')

    for data in Data.objects.all():
        if os.path.isdir(os.path.join(settings.FLOW_EXECUTOR['DATA_DIR'], str(data.id))):
            with transaction.atomic():
                # Manually set DataLocation id to preserve data directory.
                data_location = DataLocation.objects.create(id=data.id, subpath=str(data.id))
                data_location.data.add(data)

    # Increment DataLocation id's sequence
    if DataLocation.objects.exists():
        max_id = DataLocation.objects.order_by('id').last().id
        with connection.cursor() as cursor:
            cursor.execute(
                "ALTER SEQUENCE flow_datalocation_id_seq RESTART WITH {};".format(max_id + 1)
            )


Instruction:
Write a Python 3 script to
retrieve the next page of results.

Response:
def _loadMore(self, start=0, trys=0, validation=True):
        """Retrieves the next page of results."""
        self._log.debug("Loading page starting from %d" % start)
        self._czero = start
        self._pageoffs = 0
        try:
            pyxb.RequireValidWhenParsing(validation)
            self._object_list = self._client.listObjects(
                start=start,
                count=self._pagesize,
                fromDate=self._fromDate,
                nodeId=self._nodeId,
            )
        except http.client.BadStatusLine as e:
            self._log.warning("Server responded with Bad Status Line. Retrying in 5sec")
            self._client.connection.close()
            if trys > 3:
                raise e
            trys += 1
            self._loadMore(start, trys)
        except d1_common.types.exceptions.ServiceFailure as e:
            self._log.error(e)
            if trys > 3:
                raise e
            trys += 1
            self._loadMore(start, trys, validation=False)


Instruction:
Write a Python 3 script for
carving up a mime - type and returns a tuple of the type subtype and params.

Response:
def parse_mime_type(mime_type):
    """Carves up a mime-type and returns a tuple of the (type, subtype, params) where
    'params' is a dictionary of all the parameters for the media range. For example, the
    media range 'application/xhtml;q=0.5' would get parsed into:

    ('application', 'xhtml', {'q', '0.5'})

    """
    parts = mime_type.split(";")
    params = dict([tuple([s.strip() for s in param.split("=")]) for param in parts[1:]])
    full_type = parts[0].strip()
    # Java URLConnection class sends an Accept header that includes a single "*"
    # Turn it into a legal wildcard.
    if full_type == '*':
        full_type = '*/*'
    (type, subtype) = full_type.split("/")
    return (type.strip(), subtype.strip(), params)


Instruction:
Can you implement a function in Python 3 that
takes a list of supported mime - types and finds the best match for all the media - types listed in header.

Response:
def best_match(supported, header):
    """Takes a list of supported mime-types and finds the best match for all the media-
    ranges listed in header. The value of header must be a string that conforms to the
    format of the HTTP Accept: header. The value of 'supported' is a list of mime-types.

    >>> best_match(['application/xbel+xml', 'text/xml'], 'text/*;q=0.5,*/*; q=0.1')
    'text/xml'

    """
    parsed_header = [parse_media_range(r) for r in header.split(",")]
    weighted_matches = [
        (fitness_and_quality_parsed(mime_type, parsed_header), mime_type)
        for mime_type in supported
    ]
    weighted_matches.sort()
    return weighted_matches[-1][0][1] and weighted_matches[-1][1] or ''


Instruction:
Implement a Python 3 function for
chunking delete, which should be used if deleting many objects. The reason why this method is needed is that deleting a lot of Data objects requires Django to fetch all of them into memory (fast path is not used) and this causes huge memory usage (and possibly OOM). :param chunk_size: Optional chunk size

Response:
def _delete_chunked(queryset, chunk_size=500):
        """Chunked delete, which should be used if deleting many objects.

        The reason why this method is needed is that deleting a lot of Data objects
        requires Django to fetch all of them into memory (fast path is not used) and
        this causes huge memory usage (and possibly OOM).

        :param chunk_size: Optional chunk size
        """
        while True:
            # Discover primary key to limit the current chunk. This is required because delete
            # cannot be called on a sliced queryset due to ordering requirement.
            with transaction.atomic():
                # Get offset of last item (needed because it may be less than the chunk size).
                offset = queryset.order_by('pk')[:chunk_size].count()
                if not offset:
                    break

                # Fetch primary key of last item and use it to delete the chunk.
                last_instance = queryset.order_by('pk')[offset - 1]
                queryset.filter(pk__lte=last_instance.pk).delete()


Instruction:
Can you tell what is the following Python 3 function doing
def duplicate(self, contributor=None):
        bundle = [
            {'original': data, 'copy': data.duplicate(contributor=contributor)}
            for data in self
        ]

        bundle = rewire_inputs(bundle)
        duplicated = [item['copy'] for item in bundle]

        return duplicated

Response: Duplicate ( make a copy ) Data objects.


Instruction:
Explain what the following Python 3 code does
def save_storage(self, instance, schema):
        for field_schema, fields in iterate_fields(instance, schema):
            name = field_schema['name']
            value = fields[name]
            if field_schema.get('type', '').startswith('basic:json:'):
                if value and not self.pk:
                    raise ValidationError(
                        'Data object must be `created` before creating `basic:json:` fields')

                if isinstance(value, int):
                    # already in Storage
                    continue

                if isinstance(value, str):
                    file_path = self.location.get_path(filename=value)  # pylint: disable=no-member
                    if os.path.isfile(file_path):
                        try:
                            with open(file_path) as file_handler:
                                value = json.load(file_handler)
                        except json.JSONDecodeError:
                            with open(file_path) as file_handler:
                                content = file_handler.read()
                                content = content.rstrip()
                                raise ValidationError(
                                    "Value of '{}' must be a valid JSON, current: {}".format(name, content)
                                )

                storage = self.storages.create(  # pylint: disable=no-member
                    name='Storage for data id {}'.format(self.pk),
                    contributor=self.contributor,
                    json=value,
                )

                # `value` is copied by value, so `fields[name]` must be changed
                fields[name] = storage.pk

Response: Save basic : json values to a Storage collection.


Instruction:
Create a Python 3 function to
retrieve handles for all basic : secret fields on input.

Response:
def resolve_secrets(self):
        """Retrieve handles for all basic:secret: fields on input.

        The process must have the ``secrets`` resource requirement
        specified in order to access any secrets. Otherwise this method
        will raise a ``PermissionDenied`` exception.

        :return: A dictionary of secrets where key is the secret handle
            and value is the secret value.
        """
        secrets = {}
        for field_schema, fields in iterate_fields(self.input, self.process.input_schema):  # pylint: disable=no-member
            if not field_schema.get('type', '').startswith('basic:secret:'):
                continue

            name = field_schema['name']
            value = fields[name]
            try:
                handle = value['handle']
            except KeyError:
                continue

            try:
                secrets[handle] = Secret.objects.get_secret(
                    handle,
                    contributor=self.contributor
                )
            except Secret.DoesNotExist:
                raise PermissionDenied("Access to secret not allowed or secret does not exist")

        # If the process does not not have the right requirements it is not
        # allowed to access any secrets.
        allowed = self.process.requirements.get('resources', {}).get('secrets', False)  # pylint: disable=no-member
        if secrets and not allowed:
            raise PermissionDenied(
                "Process '{}' has secret inputs, but no permission to see secrets".format(
                    self.process.slug  # pylint: disable=no-member
                )
            )

        return secrets


Instruction:
Can you create a Python 3 function that
saves data and list : data references as parents.

Response:
def save_dependencies(self, instance, schema):
        """Save data: and list:data: references as parents."""
        def add_dependency(value):
            """Add parent Data dependency."""
            try:
                DataDependency.objects.update_or_create(
                    parent=Data.objects.get(pk=value),
                    child=self,
                    defaults={'kind': DataDependency.KIND_IO},
                )
            except Data.DoesNotExist:
                pass

        for field_schema, fields in iterate_fields(instance, schema):
            name = field_schema['name']
            value = fields[name]

            if field_schema.get('type', '').startswith('data:'):
                add_dependency(value)
            elif field_schema.get('type', '').startswith('list:data:'):
                for data in value:
                    add_dependency(data)


Instruction:
Implement a function in Python 3 to
create a new entity if flow_collection is defined in process.

Response:
def create_entity(self):
        """Create entity if `flow_collection` is defined in process.

        Following rules applies for adding `Data` object to `Entity`:
        * Only add `Data object` to `Entity` if process has defined
        `flow_collection` field
        * Add object to existing `Entity`, if all parents that are part
        of it (but not necessary all parents), are part of the same
        `Entity`
        * If parents belong to different `Entities` or do not belong to
        any `Entity`, create new `Entity`

        """
        entity_type = self.process.entity_type  # pylint: disable=no-member
        entity_descriptor_schema = self.process.entity_descriptor_schema  # pylint: disable=no-member
        entity_input = self.process.entity_input  # pylint: disable=no-member

        if entity_type:
            data_filter = {}
            if entity_input:
                input_id = dict_dot(self.input, entity_input, default=lambda: None)
                if input_id is None:
                    logger.warning("Skipping creation of entity due to missing input.")
                    return
                if isinstance(input_id, int):
                    data_filter['data__pk'] = input_id
                elif isinstance(input_id, list):
                    data_filter['data__pk__in'] = input_id
                else:
                    raise ValueError(
                        "Cannot create entity due to invalid value of field {}.".format(entity_input)
                    )
            else:
                data_filter['data__in'] = self.parents.all()  # pylint: disable=no-member

            entity_query = Entity.objects.filter(type=entity_type, **data_filter).distinct()
            entity_count = entity_query.count()

            if entity_count == 0:
                descriptor_schema = DescriptorSchema.objects.filter(
                    slug=entity_descriptor_schema
                ).latest()
                entity = Entity.objects.create(
                    contributor=self.contributor,
                    descriptor_schema=descriptor_schema,
                    type=entity_type,
                    name=self.name,
                    tags=self.tags,
                )
                assign_contributor_permissions(entity)

            elif entity_count == 1:
                entity = entity_query.first()
                copy_permissions(entity, self)

            else:
                logger.info("Skipping creation of entity due to multiple entities found.")
                entity = None

            if entity:
                entity.data.add(self)
                # Inherit collections from entity.
                for collection in entity.collections.all():
                    collection.data.add(self)


Instruction:
How would you code a function in Python 3 to
save the data model.

Response:
def save(self, render_name=False, *args, **kwargs):  # pylint: disable=keyword-arg-before-vararg
        """Save the data model."""
        if self.name != self._original_name:
            self.named_by_user = True

        create = self.pk is None
        if create:
            fill_with_defaults(self.input, self.process.input_schema)  # pylint: disable=no-member

            if not self.name:
                self._render_name()
            else:
                self.named_by_user = True

            self.checksum = get_data_checksum(
                self.input, self.process.slug, self.process.version)  # pylint: disable=no-member

        elif render_name:
            self._render_name()

        self.save_storage(self.output, self.process.output_schema)  # pylint: disable=no-member

        if self.status != Data.STATUS_ERROR:
            hydrate_size(self)
            # If only specified fields are updated (e.g. in executor), size needs to be added
            if 'update_fields' in kwargs:
                kwargs['update_fields'].append('size')

        # Input Data objects are validated only upon creation as they can be deleted later.
        skip_missing_data = not create
        validate_schema(
            self.input, self.process.input_schema, skip_missing_data=skip_missing_data  # pylint: disable=no-member
        )

        render_descriptor(self)

        if self.descriptor_schema:
            try:
                validate_schema(self.descriptor, self.descriptor_schema.schema)  # pylint: disable=no-member
                self.descriptor_dirty = False
            except DirtyError:
                self.descriptor_dirty = True
        elif self.descriptor and self.descriptor != {}:
            raise ValueError("`descriptor_schema` must be defined if `descriptor` is given")

        if self.status != Data.STATUS_ERROR:
            output_schema = self.process.output_schema  # pylint: disable=no-member
            if self.status == Data.STATUS_DONE:
                validate_schema(
                    self.output, output_schema, data_location=self.location, skip_missing_data=True
                )
            else:
                validate_schema(
                    self.output, output_schema, data_location=self.location, test_required=False
                )

        with transaction.atomic():
            self._perform_save(*args, **kwargs)

            # We can only save dependencies after the data object has been saved. This
            # is why a transaction block is needed and the save method must be called first.
            if create:
                self.save_dependencies(self.input, self.process.input_schema)  # pylint: disable=no-member
                self.create_entity()


Instruction:
Can you create a Python 3 function that
deletes the data model.

Response:
def delete(self, *args, **kwargs):
        """Delete the data model."""
        # Store ids in memory as relations are also deleted with the Data object.
        storage_ids = list(self.storages.values_list('pk', flat=True))  # pylint: disable=no-member

        super().delete(*args, **kwargs)

        Storage.objects.filter(pk__in=storage_ids, data=None).delete()


Instruction:
Create a Python 3 function to
create a duplicate of this data object.

Response:
def duplicate(self, contributor=None):
        """Duplicate (make a copy)."""
        if self.status not in [self.STATUS_DONE, self.STATUS_ERROR]:
            raise ValidationError('Data object must have done or error status to be duplicated')

        duplicate = Data.objects.get(id=self.id)
        duplicate.pk = None
        duplicate.slug = None
        duplicate.name = 'Copy of {}'.format(self.name)
        duplicate.duplicated = now()
        if contributor:
            duplicate.contributor = contributor

        duplicate._perform_save(force_insert=True)  # pylint: disable=protected-access

        assign_contributor_permissions(duplicate)

        # Override fields that are automatically set on create.
        duplicate.created = self.created
        duplicate._perform_save()  # pylint: disable=protected-access

        if self.location:
            self.location.data.add(duplicate)  # pylint: disable=no-member

        duplicate.storages.set(self.storages.all())  # pylint: disable=no-member

        for migration in self.migration_history.order_by('created'):  # pylint: disable=no-member
            migration.pk = None
            migration.data = duplicate
            migration.save(force_insert=True)

        # Inherit existing child dependencies.
        DataDependency.objects.bulk_create([
            DataDependency(child=duplicate, parent=dependency.parent, kind=dependency.kind)
            for dependency in DataDependency.objects.filter(child=self)
        ])
        # Inherit existing parent dependencies.
        DataDependency.objects.bulk_create([
            DataDependency(child=dependency.child, parent=duplicate, kind=dependency.kind)
            for dependency in DataDependency.objects.filter(parent=self)
        ])

        return duplicate


Instruction:
Implement a Python 3 function for
rendering data name. The rendering is based on name template (`process.data_name`) and input context.

Response:
def _render_name(self):
        """Render data name.

        The rendering is based on name template (`process.data_name`) and
        input context.

        """
        if not self.process.data_name or self.named_by_user:  # pylint: disable=no-member
            return

        inputs = copy.deepcopy(self.input)
        hydrate_input_references(inputs, self.process.input_schema, hydrate_values=False)  # pylint: disable=no-member
        template_context = inputs

        try:
            name = render_template(
                self.process,
                self.process.data_name,  # pylint: disable=no-member
                template_context
            )
        except EvaluationError:
            name = '?'

        self.name = name


Instruction:
Create a Python 3 function for
composing data location path.

Response:
def get_path(self, prefix=None, filename=None):
        """Compose data location path."""
        prefix = prefix or settings.FLOW_EXECUTOR['DATA_DIR']

        path = os.path.join(prefix, self.subpath)
        if filename:
            path = os.path.join(path, filename)

        return path


Instruction:
Can you create a Python 3 function that
composes data runtime location path.

Response:
def get_runtime_path(self, filename=None):
        """Compose data runtime location path."""
        return self.get_path(prefix=settings.FLOW_EXECUTOR['RUNTIME_DIR'], filename=filename)


Instruction:
Can you implement a function in Python 3 that
returns serialized list of data objects on entity that user has view permission on.

Response:
def get_data(self, entity):
        """Return serialized list of data objects on entity that user has `view` permission on."""
        data = self._filter_queryset('view_data', entity.data.all())

        return self._serialize_data(data)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def to_schema(self):
        process_type = self.metadata.process_type
        if not process_type.endswith(':'):
            process_type = '{}:'.format(process_type)

        schema = {
            'slug': self.metadata.slug,
            'name': self.metadata.name,
            'type': process_type,
            'version': self.metadata.version,
            'data_name': '',
            'requirements': {
                'executor': {
                    'docker': {
                        'image': 'resolwe/base:ubuntu-18.04',
                    },
                },
            },
        }

        if self.metadata.description is not None:
            schema['description'] = self.metadata.description
        if self.metadata.category is not None:
            schema['category'] = self.metadata.category
        if self.metadata.scheduling_class is not None:
            schema['scheduling_class'] = self.metadata.scheduling_class
        if self.metadata.persistence is not None:
            schema['persistence'] = self.metadata.persistence
        if self.metadata.requirements is not None:
            schema['requirements'] = self.metadata.requirements
        if self.metadata.data_name is not None:
            schema['data_name'] = self.metadata.data_name
        if self.metadata.entity is not None:
            schema['entity'] = self.metadata.entity

        if self.inputs:
            schema['input'] = []
            for field in self.inputs.values():
                schema['input'].append(field.to_schema())

        if self.outputs:
            schema['output'] = []
            for field in self.outputs.values():
                schema['output'].append(field.to_schema())

        schema['run'] = {
            'language': 'python',
            'program': self.source or '',
        }

        return schema

Response: Return the process schema for this process.


Instruction:
Can you tell what is the following Python 3 function doing
def post_register_hook(self, verbosity=1):
        if not getattr(settings, 'FLOW_DOCKER_DONT_PULL', False):
            call_command('list_docker_images', pull=True, verbosity=verbosity)

Response: Pull Docker images needed by processes after registering.


Instruction:
Can you implement a function in Python 3 that
resolves the data path for use with the executor.

Response:
def resolve_data_path(self, data=None, filename=None):
        """Resolve data path for use with the executor.

        :param data: Data object instance
        :param filename: Filename to resolve
        :return: Resolved filename, which can be used to access the
            given data file in programs executed using this executor
        """
        if data is None:
            return constants.DATA_ALL_VOLUME

        # Prefix MUST be set because ``get_path`` uses Django's settings,
        # if prefix is not set, to get path prefix. But the executor
        # shouldn't use Django's settings directly, so prefix is set
        # via a constant.
        return data.location.get_path(prefix=constants.DATA_ALL_VOLUME, filename=filename)


Instruction:
Implement a function in Python 3 to
resolve the upload path for use with the executor.

Response:
def resolve_upload_path(self, filename=None):
        """Resolve upload path for use with the executor.

        :param filename: Filename to resolve
        :return: Resolved filename, which can be used to access the
            given uploaded file in programs executed using this
            executor
        """
        if filename is None:
            return constants.UPLOAD_VOLUME

        return os.path.join(constants.UPLOAD_VOLUME, filename)


Instruction:
Implement a Python 3 function for
determining if two datetimes are equal with fuzz factor.

Response:
def are_equal(a_dt, b_dt, round_sec=1):
    """Determine if two datetimes are equal with fuzz factor.

    A naive datetime (no timezone information) is assumed to be in in UTC.

    Args:
      a_dt: datetime
        Timestamp to compare.

      b_dt: datetime
        Timestamp to compare.

      round_sec: int or float
        Round the timestamps to the closest second divisible by this value before
        comparing them.

        E.g.:

        - ``n_round_sec`` = 0.1: nearest 10th of a second.
        - ``n_round_sec`` = 1: nearest second.
        - ``n_round_sec`` = 30: nearest half minute.

        Timestamps may lose resolution or otherwise change slightly as they go through
        various transformations and storage systems. This again may cause timestamps
        that
        have been processed in different systems to fail an exact equality compare even
        if
        they were initially the same timestamp. This rounding avoids such problems as
        long
        as the error introduced to the original timestamp is not higher than the
        rounding
        value. Of course, the rounding also causes a loss in resolution in the values
        compared, so should be kept as low as possible. The default value of 1 second
        should
        be a good tradeoff in most cases.

    Returns:
      bool
        - **True**: If the two datetimes are equal after being rounded by
          ``round_sec``.

    """
    ra_dt = round_to_nearest(a_dt, round_sec)
    rb_dt = round_to_nearest(b_dt, round_sec)
    logger.debug('Rounded:')
    logger.debug('{} -> {}'.format(a_dt, ra_dt))
    logger.debug('{} -> {}'.format(b_dt, rb_dt))
    return normalize_datetime_to_utc(ra_dt) == normalize_datetime_to_utc(rb_dt)


Instruction:
Given the following Python 3 function, write the documentation
def http_datetime_str_from_dt(dt):
    epoch_seconds = ts_from_dt(dt)
    return email.utils.formatdate(epoch_seconds, localtime=False, usegmt=True)

Response: Format datetime to HTTP Full Date format.


Instruction:
Create a Python 3 function for
parsing the HTTP Full Date formats and return as datetime.

Response:
def dt_from_http_datetime_str(http_full_datetime):
    """Parse HTTP Full Date formats and return as datetime.

    Args:
      http_full_datetime : str
        Each of the allowed formats are supported:

        - Sun, 06 Nov 1994 08:49:37 GMT  ; RFC 822, updated by RFC 1123
        - Sunday, 06-Nov-94 08:49:37 GMT ; RFC 850, obsoleted by RFC 1036
        - Sun Nov  6 08:49:37 1994       ; ANSI C's asctime() format

        HTTP Full Dates are always in UTC.

    Returns:
      datetime
        The returned datetime is always timezone aware and in UTC.

    See Also:
      http://www.w3.org/Protocols/rfc2616/rfc2616-sec3.html#sec3.3.1

    """
    date_parts = list(email.utils.parsedate(http_full_datetime)[:6])
    year = date_parts[0]
    if year <= 99:
        year = year + 2000 if year < 50 else year + 1900
    return create_utc_datetime(year, *date_parts[1:])


Instruction:
Can you write a function in Python 3 where it
adjusts the datetime to UTC.

Response:
def normalize_datetime_to_utc(dt):
    """Adjust datetime to UTC.

    Apply the timezone offset to the datetime and set the timezone to UTC.

    This is a no-op if the datetime is already in UTC.

    Args:
      dt : datetime
        - tz-aware: Used in the formatted string.
        - tz-naive: Assumed to be in UTC.

    Returns:
      datetime
        The returned datetime is always timezone aware and in UTC.

    Notes:
      This forces a new object to be returned, which fixes an issue with
      serialization to XML in PyXB. PyXB uses a mixin together with
      datetime to handle the XML xs:dateTime. That type keeps track of
      timezone information included in the original XML doc, which conflicts if we
      return it here as part of a datetime mixin.

    See Also:
      ``cast_naive_datetime_to_tz()``

    """

    return datetime.datetime(
        *dt.utctimetuple()[:6], microsecond=dt.microsecond, tzinfo=datetime.timezone.utc
    )


Instruction:
Can you implement a function in Python 3 that
casts a naive datetime to a tz - aware datetime.

Response:
def cast_naive_datetime_to_tz(dt, tz=UTC()):
    """If datetime is tz-naive, set it to ``tz``. If datetime is tz-aware, return it
    unmodified.

    Args:
      dt : datetime
        tz-naive or tz-aware datetime.

      tz : datetime.tzinfo
        The timezone to which to adjust tz-naive datetime.

    Returns:
      datetime
        tz-aware datetime.

    Warning:
      This will change the actual moment in time that is represented if the datetime is
      naive and represents a date and time not in ``tz``.

    See Also:
      ``normalize_datetime_to_utc()``

    """
    if has_tz(dt):
        return dt
    return dt.replace(tzinfo=tz)


Instruction:
Can you tell what is the following Python 3 function doing
def round_to_nearest(dt, n_round_sec=1.0):
    ts = ts_from_dt(strip_timezone(dt)) + n_round_sec / 2.0
    res = dt_from_ts(ts - (ts % n_round_sec))
    return res.replace(tzinfo=dt.tzinfo)

Response: Round datetime up or down to nearest divisor.


Instruction:
How would you code a function in Python 3 to
submit a new process with SLURM.

Response:
def submit(self, data, runtime_dir, argv):
        """Run process with SLURM.

        For details, see
        :meth:`~resolwe.flow.managers.workload_connectors.base.BaseConnector.submit`.
        """
        limits = data.process.get_resource_limits()
        logger.debug(__(
            "Connector '{}' running for Data with id {} ({}).",
            self.__class__.__module__,
            data.id,
            repr(argv)
        ))

        # Compute target partition.
        partition = getattr(settings, 'FLOW_SLURM_PARTITION_DEFAULT', None)
        if data.process.slug in getattr(settings, 'FLOW_SLURM_PARTITION_OVERRIDES', {}):
            partition = settings.FLOW_SLURM_PARTITION_OVERRIDES[data.process.slug]

        try:
            # Make sure the resulting file is executable on creation.
            script_path = os.path.join(runtime_dir, 'slurm.sh')
            file_descriptor = os.open(script_path, os.O_WRONLY | os.O_CREAT, mode=0o555)
            with os.fdopen(file_descriptor, 'wt') as script:
                script.write('#!/bin/bash\n')
                script.write('#SBATCH --mem={}M\n'.format(limits['memory'] + EXECUTOR_MEMORY_OVERHEAD))
                script.write('#SBATCH --cpus-per-task={}\n'.format(limits['cores']))
                if partition:
                    script.write('#SBATCH --partition={}\n'.format(partition))

                # Render the argument vector into a command line.
                line = ' '.join(map(shlex.quote, argv))
                script.write(line + '\n')

            command = ['/usr/bin/env', 'sbatch', script_path]
            subprocess.Popen(
                command,
                cwd=runtime_dir,
                stdin=subprocess.DEVNULL
            ).wait()
        except OSError as err:
            logger.error(__(
                "OSError occurred while preparing SLURM script for Data {}: {}",
                data.id, err
            ))


Instruction:
Can you write a function in Python 3 where it
gets the list of files to purge.

Response:
def get_purge_files(root, output, output_schema, descriptor, descriptor_schema):
    """Get files to purge."""
    def remove_file(fn, paths):
        """From paths remove fn and dirs before fn in dir tree."""
        while fn:
            for i in range(len(paths) - 1, -1, -1):
                if fn == paths[i]:
                    paths.pop(i)
            fn, _ = os.path.split(fn)

    def remove_tree(fn, paths):
        """From paths remove fn and dirs before or after fn in dir tree."""
        for i in range(len(paths) - 1, -1, -1):
            head = paths[i]
            while head:
                if fn == head:
                    paths.pop(i)
                    break
                head, _ = os.path.split(head)

        remove_file(fn, paths)

    def subfiles(root):
        """Extend unreferenced list with all subdirs and files in top dir."""
        subs = []
        for path, dirs, files in os.walk(root, topdown=False):
            path = path[len(root) + 1:]
            subs.extend(os.path.join(path, f) for f in files)
            subs.extend(os.path.join(path, d) for d in dirs)
        return subs

    unreferenced_files = subfiles(root)

    remove_file('jsonout.txt', unreferenced_files)
    remove_file('stderr.txt', unreferenced_files)
    remove_file('stdout.txt', unreferenced_files)

    meta_fields = [
        [output, output_schema],
        [descriptor, descriptor_schema]
    ]

    for meta_field, meta_field_schema in meta_fields:
        for field_schema, fields in iterate_fields(meta_field, meta_field_schema):
            if 'type' in field_schema:
                field_type = field_schema['type']
                field_name = field_schema['name']

                # Remove basic:file: entries
                if field_type.startswith('basic:file:'):
                    remove_file(fields[field_name]['file'], unreferenced_files)

                # Remove list:basic:file: entries
                elif field_type.startswith('list:basic:file:'):
                    for field in fields[field_name]:
                        remove_file(field['file'], unreferenced_files)

                # Remove basic:dir: entries
                elif field_type.startswith('basic:dir:'):
                    remove_tree(fields[field_name]['dir'], unreferenced_files)

                # Remove list:basic:dir: entries
                elif field_type.startswith('list:basic:dir:'):
                    for field in fields[field_name]:
                        remove_tree(field['dir'], unreferenced_files)

                # Remove refs entries
                if field_type.startswith('basic:file:') or field_type.startswith('basic:dir:'):
                    for ref in fields[field_name].get('refs', []):
                        remove_tree(ref, unreferenced_files)

                elif field_type.startswith('list:basic:file:') or field_type.startswith('list:basic:dir:'):
                    for field in fields[field_name]:
                        for ref in field.get('refs', []):
                            remove_tree(ref, unreferenced_files)

    return set([os.path.join(root, filename) for filename in unreferenced_files])


Instruction:
Create a Python 3 function to
purge all files referenced by meta data.

Response:
def location_purge(location_id, delete=False, verbosity=0):
    """Print and conditionally delete files not referenced by meta data.

    :param location_id: Id of the
        :class:`~resolwe.flow.models.DataLocation` model that data
        objects reference to.
    :param delete: If ``True``, then delete unreferenced files.
    """
    try:
        location = DataLocation.objects.get(id=location_id)
    except DataLocation.DoesNotExist:
        logger.warning("Data location does not exist", extra={'location_id': location_id})
        return

    unreferenced_files = set()
    purged_data = Data.objects.none()
    referenced_by_data = location.data.exists()
    if referenced_by_data:
        if location.data.exclude(status__in=[Data.STATUS_DONE, Data.STATUS_ERROR]).exists():
            return

        # Perform cleanup.
        purge_files_sets = list()
        purged_data = location.data.all()
        for data in purged_data:
            purge_files_sets.append(get_purge_files(
                location.get_path(),
                data.output,
                data.process.output_schema,
                data.descriptor,
                getattr(data.descriptor_schema, 'schema', [])
            ))

        intersected_files = set.intersection(*purge_files_sets) if purge_files_sets else set()
        unreferenced_files.update(intersected_files)
    else:
        # Remove data directory.
        unreferenced_files.add(location.get_path())
        unreferenced_files.add(location.get_runtime_path())

    if verbosity >= 1:
        # Print unreferenced files
        if unreferenced_files:
            logger.info(__("Unreferenced files for location id {} ({}):", location_id, len(unreferenced_files)))
            for name in unreferenced_files:
                logger.info(__("  {}", name))
        else:
            logger.info(__("No unreferenced files for location id {}", location_id))

    # Go through unreferenced files and delete them.
    if delete:
        for name in unreferenced_files:
            if os.path.isfile(name) or os.path.islink(name):
                os.remove(name)
            elif os.path.isdir(name):
                shutil.rmtree(name)

        location.purged = True
        location.save()

        if not referenced_by_data:
            location.delete()


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _location_purge_all(delete=False, verbosity=0):
    if DataLocation.objects.exists():
        for location in DataLocation.objects.filter(Q(purged=False) | Q(data=None)):
            location_purge(location.id, delete, verbosity)
    else:
        logger.info("No data locations")

Response: Purge all data locations.


Instruction:
How would you implement a function in Python 3 that
purges all data locations and storage.

Response:
def purge_all(delete=False, verbosity=0):
    """Purge all data locations."""
    _location_purge_all(delete, verbosity)
    _storage_purge_all(delete, verbosity)


Instruction:
How would you implement a function in Python 3 that
converts value from an AST node.

Response:
def get_value(self, node):
        """Convert value from an AST node."""
        if not isinstance(node, ast.Str):
            raise TypeError("must be a string literal")

        return node.s


Instruction:
How would you code a function in Python 3 to
convert value from an AST node.

Response:
def get_value(self, node):
        """Convert value from an AST node."""
        if not isinstance(node, ast.Attribute):
            raise TypeError("must be an attribute")

        if node.value.id != self.choices.__name__:
            raise TypeError("must be an attribute of {}".format(self.choices.__name__))

        return getattr(self.choices, node.attr)


Instruction:
Can you create a Python 3 function that
converts value from an AST node.

Response:
def get_value(self, node):
        """Convert value from an AST node."""
        if not isinstance(node, ast.Dict):
            raise TypeError("must be a dictionary")

        evaluator = SafeEvaluator()
        try:
            value = evaluator.run(node)
        except Exception as ex:
            # TODO: Handle errors.
            raise ex

        try:
            # Ensure value is a serializable dictionary.
            value = json.loads(json.dumps(value))
            if not isinstance(value, dict):
                raise TypeError
        except (TypeError, ValueError):
            raise TypeError("must be serializable")

        return value


Instruction:
Write a Python 3 function that can
visit a class node containing a list of field definitions.

Response:
def visit_field_class(self, item, descriptor=None, fields=None):
        """Visit a class node containing a list of field definitions."""
        discovered_fields = collections.OrderedDict()
        field_groups = {}
        for node in item.body:
            if isinstance(node, ast.ClassDef):
                field_groups[node.name] = self.visit_field_class(node)
                continue

            if not isinstance(node, ast.Assign):
                continue
            if not isinstance(node.value, ast.Call):
                continue
            if not isinstance(node.targets[0], ast.Name):
                continue

            # Build accessible symbols table.
            symtable = {}
            # All field types.
            symtable.update({
                field.__name__: field
                for field in get_available_fields()
            })
            # Field group classes.
            symtable.update(field_groups)

            evaluator = SafeEvaluator(symtable=symtable)

            name = node.targets[0].id
            try:
                field = evaluator.run(node.value)
            except Exception as ex:
                # TODO: Handle errors.
                raise ex

            if descriptor is not None:
                field.contribute_to_class(descriptor, fields, name)
            else:
                discovered_fields[name] = field

        if descriptor is None:
            class Fields:
                """Fields wrapper."""

            for name, field in discovered_fields.items():
                setattr(Fields, name, field)

            return Fields


Instruction:
Can you tell what is the following Python 3 function doing
def visit_ClassDef(self, node):  # pylint: disable=invalid-name
        # Resolve everything as root scope contains everything from the process module.
        for base in node.bases:
            # Cover `from resolwe.process import ...`.
            if isinstance(base, ast.Name) and isinstance(base.ctx, ast.Load):
                base = getattr(runtime, base.id, None)
            # Cover `from resolwe import process`.
            elif isinstance(base, ast.Attribute) and isinstance(base.ctx, ast.Load):
                base = getattr(runtime, base.attr, None)
            else:
                continue

            if issubclass(base, runtime.Process):
                break
        else:
            return

        descriptor = ProcessDescriptor(source=self.source)

        # Available embedded classes.
        embedded_class_fields = {
            runtime.PROCESS_INPUTS_NAME: descriptor.inputs,
            runtime.PROCESS_OUTPUTS_NAME: descriptor.outputs,
        }

        # Parse metadata in class body.
        for item in node.body:
            if isinstance(item, ast.Assign):
                # Possible metadata.
                if (len(item.targets) == 1 and isinstance(item.targets[0], ast.Name)
                        and isinstance(item.targets[0].ctx, ast.Store)
                        and item.targets[0].id in PROCESS_METADATA):
                    # Try to get the metadata value.
                    value = PROCESS_METADATA[item.targets[0].id].get_value(item.value)
                    setattr(descriptor.metadata, item.targets[0].id, value)
            elif (isinstance(item, ast.Expr) and isinstance(item.value, ast.Str)
                  and descriptor.metadata.description is None):
                # Possible description string.
                descriptor.metadata.description = item.value.s
            elif isinstance(item, ast.ClassDef) and item.name in embedded_class_fields.keys():
                # Possible input/output declaration.
                self.visit_field_class(item, descriptor, embedded_class_fields[item.name])

        descriptor.validate()
        self.processes.append(descriptor)

Response: Visit the class definition node.


Instruction:
Explain what the following Python 3 code does
def parse(self):
        root = ast.parse(self._source)

        visitor = ProcessVisitor(source=self._source)
        visitor.visit(root)
        return visitor.processes

Response:
Parse process.

        :return: A list of discovered process descriptors


Instruction:
Can you tell what is the following Python 3 function doing
def get_permissions_class(permissions_name=None):
    def load_permissions(permissions_name):
        """Look for a fully qualified flow permissions class."""
        try:
            return import_module('{}'.format(permissions_name)).ResolwePermissions
        except AttributeError:
            raise AttributeError("'ResolwePermissions' class not found in {} module.".format(
                permissions_name))
        except ImportError as ex:
            # The permissions module wasn't found. Display a helpful error
            # message listing all possible (built-in) permissions classes.
            permissions_dir = os.path.join(os.path.dirname(upath(__file__)), '..', 'perms')
            permissions_dir = os.path.normpath(permissions_dir)

            try:
                builtin_permissions = [
                    name for _, name, _ in pkgutil.iter_modules([permissions_dir]) if name not in ['tests']]
            except EnvironmentError:
                builtin_permissions = []
            if permissions_name not in ['resolwe.auth.{}'.format(p) for p in builtin_permissions]:
                permissions_reprs = map(repr, sorted(builtin_permissions))
                err_msg = ("{} isn't an available flow permissions class.\n"
                           "Try using 'resolwe.auth.XXX', where XXX is one of:\n"
                           "    {}\n"
                           "Error was: {}".format(permissions_name, ", ".join(permissions_reprs), ex))
                raise ImproperlyConfigured(err_msg)
            else:
                # If there's some other error, this must be an error in Django
                raise

    if permissions_name is None:
        permissions_name = settings.FLOW_API['PERMISSIONS']

    if permissions_name not in permissions_classes:
        permissions_classes[permissions_name] = load_permissions(permissions_name)

    return permissions_classes[permissions_name]

Response: Load and cache permissions class.


Instruction:
Here you have a function in Python 3, explain what it does
def get_identifiers(sysmeta_pyxb):
    pid = d1_common.xml.get_opt_val(sysmeta_pyxb, 'identifier')
    sid = d1_common.xml.get_opt_val(sysmeta_pyxb, 'seriesId')
    obsoletes_pid = d1_common.xml.get_opt_val(sysmeta_pyxb, 'obsoletes')
    obsoleted_by_pid = d1_common.xml.get_opt_val(sysmeta_pyxb, 'obsoletedBy')
    return pid, sid, obsoletes_pid, obsoleted_by_pid

Response: Get set of identifiers that provide revision context for SciObj.


Instruction:
Write a Python 3 function that can
sort objects by dependency.

Response:
def topological_sort(unsorted_dict):
    """Sort objects by dependency.

    Sort a dict of obsoleting PID to obsoleted PID to a list of PIDs in order of
    obsolescence.

    Args:
      unsorted_dict : dict
        Dict that holds obsolescence information. Each ``key/value`` pair establishes
        that the PID in ``key`` identifies an object that obsoletes an object identifies
        by the PID in ``value``.

    Returns:
      tuple of sorted_list, unconnected_dict :

      ``sorted_list``: A list of PIDs ordered so that all PIDs that obsolete an object
      are listed after the object they obsolete.

      ``unconnected_dict``: A dict of PID to obsoleted PID of any objects that could not
      be added to a revision chain. These items will have obsoletes PIDs that directly
      or indirectly reference a PID that could not be sorted.

    Notes:
      ``obsoletes_dict`` is modified by the sort and on return holds any items that
      could not be sorted.

      The sort works by repeatedly iterating over an unsorted list of PIDs and
      moving PIDs to the sorted list as they become available. A PID is available to
      be moved to the sorted list if it does not obsolete a PID or if the PID it
      obsoletes is already in the sorted list.

    """
    sorted_list = []
    sorted_set = set()
    found = True
    unconnected_dict = unsorted_dict.copy()
    while found:
        found = False
        for pid, obsoletes_pid in list(unconnected_dict.items()):
            if obsoletes_pid is None or obsoletes_pid in sorted_set:
                found = True
                sorted_list.append(pid)
                sorted_set.add(pid)
                del unconnected_dict[pid]
    return sorted_list, unconnected_dict


Instruction:
How would you implement a function in Python 3 that
returns a list of all PIDs in the revision chain.

Response:
def get_pids_in_revision_chain(client, did):
    """Args: client: d1_client.cnclient.CoordinatingNodeClient or
    d1_client.mnclient.MemberNodeClient.

      did : str
        SID or a PID of any object in a revision chain.

    Returns:
      list of str:
        All PIDs in the chain. The returned list is in the same order as the chain. The
        initial PID is typically obtained by resolving a SID. If the given PID is not in
        a chain, a list containing the single object is returned.

    """

    def _req(p):
        return d1_common.xml.get_req_val(p)

    def _opt(p, a):
        return d1_common.xml.get_opt_val(p, a)

    sysmeta_pyxb = client.getSystemMetadata(did)
    # Walk to tail
    while _opt(sysmeta_pyxb, 'obsoletes'):
        sysmeta_pyxb = client.getSystemMetadata(_opt(sysmeta_pyxb, 'obsoletes'))
    chain_pid_list = [_req(sysmeta_pyxb.identifier)]
    # Walk from tail to head, recording traversed PIDs
    while _opt(sysmeta_pyxb, 'obsoletedBy'):
        sysmeta_pyxb = client.getSystemMetadata(_opt(sysmeta_pyxb, 'obsoletedBy'))
        chain_pid_list.append(_req(sysmeta_pyxb.identifier))
    return chain_pid_list


Instruction:
Make a summary of the following Python 3 code
def main():
    parser = argparse.ArgumentParser(
        description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
    )
    parser.add_argument("path", nargs="+", help="File or directory path")
    parser.add_argument("--exclude", nargs="+", help="Exclude glob patterns")
    parser.add_argument(
        "--no-recursive",
        dest="recursive",
        action="store_false",
        help="Search directories recursively",
    )
    parser.add_argument(
        "--ignore-invalid", action="store_true", help="Ignore invalid paths"
    )
    parser.add_argument(
        "--pycharm", action="store_true", help="Enable PyCharm integration"
    )
    parser.add_argument(
        "--diff",
        dest="show_diff",
        action="store_true",
        help="Show diff and do not modify any files",
    )
    parser.add_argument(
        "--dry-run", action="store_true", help="Process files but do not write results"
    )
    parser.add_argument("--debug", action="store_true", help="Debug level logging")

    args = parser.parse_args()
    d1_common.util.log_setup(args.debug)

    repo_path = d1_dev.util.find_repo_root_by_path(__file__)
    repo = git.Repo(repo_path)

    specified_file_path_list = get_specified_file_path_list(args)
    tracked_path_list = list(d1_dev.util.get_tracked_files(repo))
    format_path_list = sorted(
        set(specified_file_path_list).intersection(tracked_path_list)
    )

    progress_logger.start_task_type("Format modules", len(format_path_list))

    for i, format_path in enumerate(format_path_list):
        progress_logger.start_task("Format modules", i)
        format_all_docstr(args, format_path)

    progress_logger.end_task_type("Format modules")

Response:
This function is called by the command line interface to remove unused imports Unsafe!
    Only tested on our codebase which uses simple absolute imports on the form import
    a. b. c.


Instruction:
Explain what the following Python 3 code does
def get_docstr_list(red):
    docstr_list = []
    for n in red.find_all("string"):
        if n.value.startswith('"""'):
            docstr_list.append(n)
    return docstr_list

Response: Find all triple - quoted docstrings in module.


Instruction:
Can you create a Python 3 function that
unwraps a string into a list of consequtive blocks.

Response:
def unwrap(s, node_indent):
    """Group lines of a docstring to blocks.

    For now, only groups markdown list sections.

    A block designates a list of consequtive lines that all start at the same
    indentation level.

    The lines of the docstring are iterated top to bottom. Each line is added to
    `block_list` until a line is encountered that breaks sufficiently with the previous
    line to be deemed to be the start of a new block. At that point, all lines
    currently
    in `block_list` are stripped and joined to a single line, which is added to
    `unwrap_list`.

    Some of the block breaks are easy to determine. E.g., a line that starts with "- "
    is the start of a new markdown style list item, so is always the start of a new
    block. But then there are things like this, which is a single block:

    - An example list with a second line

    And this, which is 3 single line blocks (due to the different indentation levels):

    Args:
      jwt_bu64: bytes
        JWT, encoded using a a URL safe flavor of Base64.

    """

    def get_indent():
        if line_str.startswith('"""'):
            return node_indent
        return len(re.match(r"^( *)", line_str).group(1))

    def finish_block():
        if block_list:
            unwrap_list.append(
                (block_indent, (" ".join([v.strip() for v in block_list])).strip())
            )
            block_list.clear()

    unwrap_list = []

    block_indent = None
    block_list = []

    for line_str in s.splitlines():
        line_str = line_str.rstrip()
        line_indent = get_indent()

        # A new block has been started. Record the indent of the first line in that
        # block to use as the indent for all the lines that will be put in this block.
        if not block_list:
            block_indent = line_indent

        # A blank line always starts a new block.
        if line_str == "":
            finish_block()

        # Indent any lines that are less indentend than the docstr node
        # if line_indent < node_indent:
        #     line_indent = block_indent

        # A line that is indented less than the current block starts a new block.
        if line_indent < block_indent:
            finish_block()

        # A line that is the start of a markdown list starts a new block.
        elif line_str.strip().startswith(("- ", "* ")):
            finish_block()

        # A markdown title always starts a new block.
        elif line_str.strip().endswith(":"):
            finish_block()

        block_list.append(line_str)

        # Only make blocks for markdown list items. Write everything else as single line items.
        if not block_list[0].strip().startswith(("- ", "* ")):
            finish_block()

    # Finish the block that was in progress when the end of the docstring was reached.
    finish_block()

    return unwrap_list


Instruction:
Write a Python 3 function that can
wrap a single line to one or more lines that start at indent_int and end at indent_int and force a break at the end of the last word that will fit before WRAP_MARGIN_INT.

Response:
def wrap(indent_int, unwrap_str):
    """Wrap a single line to one or more lines that start at indent_int and end at the
    last word that will fit before WRAP_MARGIN_INT.

    If there are no word breaks (spaces) before WRAP_MARGIN_INT, force a break at
    WRAP_MARGIN_INT.

    """
    with io.StringIO() as str_buf:
        is_rest_block = unwrap_str.startswith(("- ", "* "))

        while unwrap_str:
            cut_pos = (unwrap_str + " ").rfind(" ", 0, WRAP_MARGIN_INT - indent_int)

            if cut_pos == -1:
                cut_pos = WRAP_MARGIN_INT

            this_str, unwrap_str = unwrap_str[:cut_pos], unwrap_str[cut_pos + 1 :]
            str_buf.write("{}{}\n".format(" " * indent_int, this_str))

            if is_rest_block:
                is_rest_block = False
                indent_int += 2

        return str_buf.getvalue()


Instruction:
Can you write a function in Python 3 where it
creates a simple OAI -ORE Resource Map with one Science Metadata document and any number of Science Data objects.

Response:
def createSimpleResourceMap(ore_pid, scimeta_pid, sciobj_pid_list):
    """Create a simple OAI-ORE Resource Map with one Science Metadata document and any
    number of Science Data objects.

    This creates a document that establishes an association between a Science Metadata
    object and any number of Science Data objects. The Science Metadata object contains
    information that is indexed by DataONE, allowing both the Science Metadata and the
    Science Data objects to be discoverable in DataONE Search. In search results, the
    objects will appear together and can be downloaded as a single package.

    Args:
      ore_pid: str
        Persistent Identifier (PID) to use for the new Resource Map

      scimeta_pid: str
        PID for an object that will be listed as the Science Metadata that is
        describing the Science Data objects.

      sciobj_pid_list: list of str
        List of PIDs that will be listed as the Science Data objects that are being
        described by the Science Metadata.

    Returns:
      ResourceMap : OAI-ORE Resource Map

    """
    ore = ResourceMap()
    ore.initialize(ore_pid)
    ore.addMetadataDocument(scimeta_pid)
    ore.addDataDocuments(sciobj_pid_list, scimeta_pid)
    return ore


Instruction:
Can you write a function in Python 3 where it
creates a simple OAI -ORE Resource Map from a stream of PIDs.

Response:
def createResourceMapFromStream(in_stream, base_url=d1_common.const.URL_DATAONE_ROOT):
    """Create a simple OAI-ORE Resource Map with one Science Metadata document and any
    number of Science Data objects, using a stream of PIDs.

    Args:
      in_stream:
        The first non-blank line is the PID of the resource map itself. Second line is
        the science metadata PID and remaining lines are science data PIDs.

        Example stream contents:

        ::

          PID_ORE_value
          sci_meta_pid_value
          data_pid_1
          data_pid_2
          data_pid_3

      base_url : str
        Root of the DataONE environment in which the Resource Map will be used.

    Returns:
      ResourceMap : OAI-ORE Resource Map

    """
    pids = []
    for line in in_stream:
        pid = line.strip()
        if pid == "#" or pid.startswith("# "):
            continue

    if len(pids) < 2:
        raise ValueError("Insufficient numbers of identifiers provided.")

    logging.info("Read {} identifiers".format(len(pids)))
    ore = ResourceMap(base_url=base_url)
    logging.info("ORE PID = {}".format(pids[0]))
    ore.initialize(pids[0])
    logging.info("Metadata PID = {}".format(pids[1]))
    ore.addMetadataDocument(pids[1])
    ore.addDataDocuments(pids[2:], pids[1])

    return ore


Instruction:
How would you explain what the following Python 3 function does
def initialize(self, pid, ore_software_id=d1_common.const.ORE_SOFTWARE_ID):
        # Set nice prefixes for the namespaces
        for k in list(d1_common.const.ORE_NAMESPACE_DICT.keys()):
            self.bind(k, d1_common.const.ORE_NAMESPACE_DICT[k])
        # Create the ORE entity
        oid = self._pid_to_id(pid)
        ore = rdflib.URIRef(oid)
        self.add((ore, rdflib.RDF.type, ORE.ResourceMap))
        self.add((ore, DCTERMS.identifier, rdflib.term.Literal(pid)))
        self.add((ore, DCTERMS.creator, rdflib.term.Literal(ore_software_id)))
        # Add an empty aggregation
        ag = rdflib.URIRef(oid + "#aggregation")
        self.add((ore, ORE.describes, ag))
        self.add((ag, rdflib.RDF.type, ORE.Aggregation))
        self.add((ORE.Aggregation, rdflib.RDFS.isDefinedBy, ORE.term("")))
        self.add(
            (ORE.Aggregation, rdflib.RDFS.label, rdflib.term.Literal("Aggregation"))
        )
        self._ore_initialized = True

Response: Create the basic ORE document structure.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def serialize_to_transport(self, doc_format="xml", *args, **kwargs):
        return super(ResourceMap, self).serialize(
            format=doc_format, encoding="utf-8", *args, **kwargs
        )

Response: Serialize to UTF - 8 encoded XML document.


Instruction:
Implement a function in Python 3 to
serialize ResourceMap to an XML doc that is pretty printed for display.

Response:
def serialize_to_display(self, doc_format="pretty-xml", *args, **kwargs):
        """Serialize ResourceMap to an XML doc that is pretty printed for display.

        Args:
          doc_format: str
            One of: ``xml``, ``n3``, ``turtle``, ``nt``, ``pretty-xml``, ``trix``,
            ``trig`` and ``nquads``.

          args and kwargs:
            Optional arguments forwarded to rdflib.ConjunctiveGraph.serialize().

        Returns:
          str: Pretty printed Resource Map XML doc

        Note:
          Only the default, "xml", is automatically indexed by DataONE.

        """
        return (
            super(ResourceMap, self)
            .serialize(format=doc_format, encoding=None, *args, **kwargs)
            .decode("utf-8")
        )


Instruction:
Create a Python 3 function to
return the URIRef of the Aggregation entity

Response:
def getAggregation(self):
        """Returns:

        str : URIRef of the Aggregation entity

        """
        self._check_initialized()
        return [
            o for o in self.subjects(predicate=rdflib.RDF.type, object=ORE.Aggregation)
        ][0]


Instruction:
Explain what the following Python 3 code does
def getObjectByPid(self, pid):
        self._check_initialized()
        opid = rdflib.term.Literal(pid)
        res = [o for o in self.subjects(predicate=DCTERMS.identifier, object=opid)]
        return res[0]

Response: Returns the object identified by pid.


Instruction:
Explain what the following Python 3 code does
def addResource(self, pid):
        self._check_initialized()
        try:
            # is entry already in place?
            self.getObjectByPid(pid)
            return
        except IndexError:
            pass
        # Entry not present, add it to the graph
        oid = self._pid_to_id(pid)
        obj = rdflib.URIRef(oid)
        ag = self.getAggregation()
        self.add((ag, ORE.aggregates, obj))
        self.add((obj, ORE.isAggregatedBy, ag))
        self.add((obj, DCTERMS.identifier, rdflib.term.Literal(pid)))

Response: Add a resource to the Resource Map.


Instruction:
Here you have a function in Python 3, explain what it does
def setDocuments(self, documenting_pid, documented_pid):
        self._check_initialized()
        documenting_id = self.getObjectByPid(documenting_pid)
        documented_id = self.getObjectByPid(documented_pid)
        self.add((documenting_id, CITO.documents, documented_id))

Response:
Add a CiTO the Citation Typing Ontology triple asserting that
        documenting_pid documents documented_pid


Instruction:
Explain what the following Python 3 code does
def setDocumentedBy(self, documented_pid, documenting_pid):
        self._check_initialized()
        documented_id = self.getObjectByPid(documented_pid)
        documenting_id = self.getObjectByPid(documenting_pid)
        self.add((documented_id, CITO.isDocumentedBy, documenting_id))

Response: Add a CiTO the Citation Typing Ontology triple asserting that documented_pid is documenting_pid.


Instruction:
Write a Python 3 script for
adding a list of PIDs of Science Data objects to the list of Science Data objects that documents the Science Data objects.

Response:
def addDataDocuments(self, scidata_pid_list, scimeta_pid=None):
        """Add Science Data object(s)

        Args:
          scidata_pid_list : list of str
            List of one or more PIDs of Science Data objects

          scimeta_pid: str
            PID of a Science Metadata object that documents the Science Data objects.

        """
        mpids = self.getAggregatedScienceMetadataPids()
        if scimeta_pid is None:
            if len(mpids) > 1:
                raise ValueError(
                    "No metadata PID specified and more than one choice available."
                )
            scimeta_pid = mpids[0]
        else:
            if scimeta_pid not in mpids:
                self.addMetadataDocument(scimeta_pid)

        for dpid in scidata_pid_list:
            self.addResource(dpid)
            self.setDocumentedBy(dpid, scimeta_pid)
            self.setDocuments(scimeta_pid, dpid)


Instruction:
Create a Python 3 function to
return the PID of the Resource Map itself.

Response:
def getResourceMapPid(self):
        """Returns:

        str : PID of the Resource Map itself.

        """
        ore = [
            o for o in self.subjects(predicate=rdflib.RDF.type, object=ORE.ResourceMap)
        ][0]
        pid = [str(o) for o in self.objects(predicate=DCTERMS.identifier, subject=ore)][
            0
        ]
        return pid


Instruction:
Here you have a function in Python 3, explain what it does
def getAllTriples(self):
        return [(str(s), str(p), str(o)) for s, p, o in self]

Response: Returns a list of tuples containing a subject predicate object.


Instruction:
How would you explain what the following Python 3 function does
def getSubjectObjectsByPredicate(self, predicate):
        return sorted(
            set(
                [
                    (str(s), str(o))
                    for s, o in self.subject_objects(rdflib.term.URIRef(predicate))
                ]
            )
        )

Response: Returns a list of subject and object tuples that match the predicate.


Instruction:
How would you code a function in Python 3 to
parse a OAI -ORE resource maps document.

Response:
def parseDoc(self, doc_str, format="xml"):
        """Parse a OAI-ORE Resource Maps document.

        See Also:   ``rdflib.ConjunctiveGraph.parse`` for documentation on arguments.

        """
        self.parse(data=doc_str, format=format)
        self._ore_initialized = True
        return self


Instruction:
Explain what the following Python 3 code does
def _pid_to_id(self, pid):
        return d1_common.url.joinPathElements(
            self._base_url,
            self._version_tag,
            "resolve",
            d1_common.url.encodePathElement(pid),
        )

Response: Converts a pid to a URI that can be used as an OAI -ORE identifier.


Instruction:
Write a Python 3 function for
making the script required for checking checksums from another machine.

Response:
def make_checksum_validation_script(stats_list):
    """Make batch files required for checking checksums from another machine."""
    if not os.path.exists('./hash_check'):
        os.mkdir('./hash_check')

    with open('./hash_check/curl.sh', 'w') as curl_f, open(
        './hash_check/md5.txt', 'w'
    ) as md5_f, open('./hash_check/sha1.txt', 'w') as sha1_f:

        curl_f.write('#!/usr/bin/env bash\n\n')

        for stats_dict in stats_list:
            for sysmeta_xml in stats_dict['largest_sysmeta_xml']:
                print(sysmeta_xml)
                sysmeta_pyxb = d1_common.types.dataoneTypes_v1_2.CreateFromDocument(
                    sysmeta_xml
                )

                pid = sysmeta_pyxb.identifier.value().encode('utf-8')
                file_name = re.sub('\W+', '_', pid)
                size = sysmeta_pyxb.size
                base_url = stats_dict['gmn_dict']['base_url']

                if size > 100 * 1024 * 1024:
                    logging.info('Ignored large object. size={} pid={}')

                curl_f.write('# {} {}\n'.format(size, pid))
                curl_f.write(
                    'curl -o obj/{} {}/v1/object/{}\n'.format(
                        file_name, base_url, d1_common.url.encodePathElement(pid)
                    )
                )

                if sysmeta_pyxb.checksum.algorithm == 'MD5':
                    md5_f.write(
                        '{} obj/{}\n'.format(sysmeta_pyxb.checksum.value(), file_name)
                    )
                else:
                    sha1_f.write(
                        '{} obj/{}\n'.format(sysmeta_pyxb.checksum.value(), file_name)
                    )

    with open('./hash_check/check.sh', 'w') as f:
        f.write('#!/usr/bin/env bash\n\n')
        f.write('mkdir -p obj\n')
        f.write('./curl.sh\n')
        f.write('sha1sum -c sha1.txt\n')
        f.write('md5sum -c md5.txt\n')


Instruction:
How would you explain what the following Python 3 function does
def formfield(self, **kwargs):
        kwargs.update({
            'max_value': 90,
            'min_value': -90,
        })
        return super(LatitudeField, self).formfield(**kwargs)

Response: Returns a formfield with max_value 90 and min_value - 90.


Instruction:
Given the following Python 3 function, write the documentation
def formfield(self, **kwargs):
        kwargs.update({
            'max_value': 180,
            'min_value': -180,
        })
        return super(LongitudeField, self).formfield(**kwargs)

Response: Returns a formfield with max_value 180 and min_value - 180.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def formfield(self, **kwargs):
        widget_kwargs = {
            'lat_field': self.latitude_field_name,
            'lon_field': self.longitude_field_name,
        }

        if self.data_field_name:
            widget_kwargs['data_field'] = self.data_field_name

        defaults = {
            'form_class': OSMFormField,
            'widget': OSMWidget(**widget_kwargs),
        }
        defaults.update(kwargs)

        return super(OSMField, self).formfield(**defaults)

Response: Returns a new OSMFormField with the specified kwargs.


Instruction:
Can you tell what is the following Python 3 function doing
def longitude_field_name(self):
        if self._lon_field_name is None:
            self._lon_field_name = self.name + '_lon'
        return self._lon_field_name

Response: The name of the related : class : LongitudeField.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def update_dependency_kinds(apps, schema_editor):
    DataDependency = apps.get_model('flow', 'DataDependency')
    for dependency in DataDependency.objects.all():
        # Assume dependency is of subprocess kind.
        dependency.kind = 'subprocess'

        # Check child inputs to determine if this is an IO dependency.
        child = dependency.child
        parent = dependency.parent

        for field_schema, fields in iterate_fields(child.input, child.process.input_schema):
            name = field_schema['name']
            value = fields[name]

            if field_schema.get('type', '').startswith('data:'):
                if value == parent.pk:
                    dependency.kind = 'io'
                    break
            elif field_schema.get('type', '').startswith('list:data:'):
                for data in value:
                    if value == parent.pk:
                        dependency.kind = 'io'
                        break

        dependency.save()

Response: Update historical dependency kinds as they may be wrong.


Instruction:
Given the following Python 3 function, write the documentation
def _wrap_jinja_filter(self, function):
        def wrapper(*args, **kwargs):
            """Filter wrapper."""
            try:
                return function(*args, **kwargs)
            except Exception:  # pylint: disable=broad-except
                return NestedUndefined()

        # Copy over Jinja filter decoration attributes.
        for attribute in dir(function):
            if attribute.endswith('filter'):
                setattr(wrapper, attribute, getattr(function, attribute))

        return wrapper

Response: Wrap a function to propagate exceptions as undefined values filter.


Instruction:
Explain what the following Python 3 code does
def _register_custom_filters(self):
        custom_filters = self.settings.get('CUSTOM_FILTERS', [])
        if not isinstance(custom_filters, list):
            raise KeyError("`CUSTOM_FILTERS` setting must be a list.")

        for filter_module_name in custom_filters:
            try:
                filter_module = import_module(filter_module_name)
            except ImportError as error:
                raise ImproperlyConfigured(
                    "Failed to load custom filter module '{}'.\n"
                    "Error was: {}".format(filter_module_name, error)
                )

            try:
                filter_map = getattr(filter_module, 'filters')
                if not isinstance(filter_map, dict):
                    raise TypeError
            except (AttributeError, TypeError):
                raise ImproperlyConfigured(
                    "Filter module '{}' does not define a 'filters' dictionary".format(filter_module_name)
                )
            self._environment.filters.update(filter_map)

Response: Register any custom filter modules.


Instruction:
Implement a Python 3 function for
configuring the evaluation context.

Response:
def _evaluation_context(self, escape, safe_wrapper):
        """Configure the evaluation context."""
        self._escape = escape
        self._safe_wrapper = safe_wrapper

        try:
            yield
        finally:
            self._escape = None
            self._safe_wrapper = None


Instruction:
Write a Python 3 function for
evaluating a template block.

Response:
def evaluate_block(self, template, context=None, escape=None, safe_wrapper=None):
        """Evaluate a template block."""
        if context is None:
            context = {}

        try:
            with self._evaluation_context(escape, safe_wrapper):
                template = self._environment.from_string(template)
                return template.render(**context)
        except jinja2.TemplateError as error:
            raise EvaluationError(error.args[0])
        finally:
            self._escape = None


Instruction:
Given the following Python 3 function, write the documentation
def evaluate_inline(self, expression, context=None, escape=None, safe_wrapper=None):
        if context is None:
            context = {}

        try:
            with self._evaluation_context(escape, safe_wrapper):
                compiled = self._environment.compile_expression(expression)
                return compiled(**context)
        except jinja2.TemplateError as error:
            raise EvaluationError(error.args[0])

Response: Evaluate an inline expression.


Instruction:
Explain what the following Python 3 code does
def update_module_file(redbaron_tree, module_path, show_diff=False, dry_run=False):
    with tempfile.NamedTemporaryFile() as tmp_file:
        tmp_file.write(redbaron_tree_to_module_str(redbaron_tree))
        tmp_file.seek(0)
        if are_files_equal(module_path, tmp_file.name):
            logging.debug('Source unchanged')
            return False

        logging.debug('Source modified')

        tmp_file.seek(0)
        diff_update_file(module_path, tmp_file.read(), show_diff, dry_run)

Response: Update the module_path with the contents of the tree.


Instruction:
Given the following Python 3 function, write the documentation
def find_repo_root_by_path(path):
    repo = git.Repo(path, search_parent_directories=True)
    repo_path = repo.git.rev_parse('--show-toplevel')
    logging.info('Repository: {}'.format(repo_path))
    return repo_path

Response: Given a path to an item in a git repository find the root of the repository.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _format_value(self, operation, key, indent):
        v = self._find_value(operation, key)
        if v == "NOT_FOUND":
            return []
        if not isinstance(v, list):
            v = [v]
        if not len(v):
            v = [None]
        key = key + ":"
        lines = []
        for s in v:
            # Access control rules are stored in tuples.
            if isinstance(s, tuple):
                s = "{}: {}".format(*s)
            lines.append(
                "{}{}{}{}".format(
                    " " * indent, key, " " * (TAB - indent - len(key) - 1), s
                )
            )
            key = ""
        return lines

Response: Format the value of the key in the operation.


Instruction:
Given the following Python 3 function, write the documentation
def add_cors_headers(response, request):
    opt_method_list = ",".join(request.allowed_method_list + ["OPTIONS"])
    response["Allow"] = opt_method_list
    response["Access-Control-Allow-Methods"] = opt_method_list
    response["Access-Control-Allow-Origin"] = request.META.get("Origin", "*")
    response["Access-Control-Allow-Headers"] = "Authorization"
    response["Access-Control-Allow-Credentials"] = "true"

Response: Add CORS headers to response.


Instruction:
Can you generate the documentation for the following Python 3 function
def run_process(self, slug, inputs):
        def export_files(value):
            """Export input files of spawned process."""
            if isinstance(value, str) and os.path.isfile(value):
                # TODO: Use the protocol to export files and get the
                # process schema to check field type.
                print("export {}".format(value))
            elif isinstance(value, dict):
                for item in value.values():
                    export_files(item)
            elif isinstance(value, list):
                for item in value:
                    export_files(item)

        export_files(inputs)
        print('run {}'.format(json.dumps({'process': slug, 'input': inputs}, separators=(',', ':'))))

Response: Run a new process from a running process.


Instruction:
Create a Python 3 function to
find data object ID for given slug.

Response:
def get_data_id_by_slug(self, slug):
        """Find data object ID for given slug.

        This method queries the Resolwe API and requires network access.
        """
        resolwe_host = os.environ.get('RESOLWE_HOST_URL')
        url = urllib.parse.urljoin(resolwe_host, '/api/data?slug={}&fields=id'.format(slug))

        with urllib.request.urlopen(url, timeout=60) as f:
            data = json.loads(f.read().decode('utf-8'))

        if len(data) == 1:
            return data[0]['id']
        elif not data:
            raise ValueError('Data not found for slug {}'.format(slug))
        else:
            raise ValueError('More than one data object returned for slug {}'.format(slug))


Instruction:
Implement a function in Python 3 to
return a dictionary of the requirements of the current object.

Response:
def requirements(self):
        """Process requirements."""
        class dotdict(dict):  # pylint: disable=invalid-name
            """Dot notation access to dictionary attributes."""

            def __getattr__(self, attr):
                value = self.get(attr)
                return dotdict(value) if isinstance(value, dict) else value

        return dotdict(self._meta.metadata.requirements)


Instruction:
Explain what the following Python 3 code does
def start(self, inputs):
        self.logger.info("Process is starting")

        outputs = Outputs(self._meta.outputs)

        self.logger.info("Process is running")
        try:
            self.run(inputs.freeze(), outputs)
            return outputs.freeze()
        except Exception as error:
            self.logger.exception("Exception while running process")
            print(resolwe_runtime_utils.error(str(error)))
            raise
        except:  # noqa
            self.logger.exception("Exception while running process")
            print(resolwe_runtime_utils.error("Exception while running process"))
            raise
        finally:
            self.logger.info("Process has finished")

Response:
Start the process.

        :param inputs: An instance of `Inputs` describing the process inputs
        :return: An instance of `Outputs` describing the process outputs


Instruction:
Write a Python 3 function that can
perform process discovery in given path and return a list of process schemas.

Response:
def discover_process(self, path):
        """Perform process discovery in given path.

        This method will be called during process registration and
        should return a list of dictionaries with discovered process
        schemas.
        """
        if not path.lower().endswith('.py'):
            return []

        parser = SafeParser(open(path).read())
        processes = parser.parse()
        return [process.to_schema() for process in processes]


Instruction:
Create a Python 3 function to
evaluate the code needed to compute a given Data object.

Response:
def evaluate(self, data):
        """Evaluate the code needed to compute a given Data object."""
        return 'PYTHONPATH="{runtime}" python3 -u -m resolwe.process {program} --slug {slug} --inputs {inputs}'.format(
            runtime=PYTHON_RUNTIME_VOLUME,
            program=PYTHON_PROGRAM_VOLUME,
            slug=shlex.quote(data.process.slug),
            inputs=PYTHON_INPUTS_VOLUME,
        )


Instruction:
Given the following Python 3 function, write the documentation
def prepare_runtime(self, runtime_dir, data):
        # Copy over Python process runtime (resolwe.process).
        import resolwe.process as runtime_package

        src_dir = os.path.dirname(inspect.getsourcefile(runtime_package))
        dest_package_dir = os.path.join(runtime_dir, PYTHON_RUNTIME_DIRNAME, 'resolwe', 'process')
        shutil.copytree(src_dir, dest_package_dir)
        os.chmod(dest_package_dir, 0o755)

        # Write python source file.
        source = data.process.run.get('program', '')
        program_path = os.path.join(runtime_dir, PYTHON_PROGRAM_FILENAME)
        with open(program_path, 'w') as file:
            file.write(source)
        os.chmod(program_path, 0o755)

        # Write serialized inputs.
        inputs = copy.deepcopy(data.input)
        hydrate_input_references(inputs, data.process.input_schema)
        hydrate_input_uploads(inputs, data.process.input_schema)
        inputs_path = os.path.join(runtime_dir, PYTHON_INPUTS_FILENAME)

        # XXX: Skip serialization of LazyStorageJSON. We should support
        # LazyStorageJSON in Python processes on the new communication protocol
        def default(obj):
            """Get default value."""
            class_name = obj.__class__.__name__
            if class_name == 'LazyStorageJSON':
                return ''

            raise TypeError(f'Object of type {class_name} is not JSON serializable')

        with open(inputs_path, 'w') as file:
            json.dump(inputs, file, default=default)

        # Generate volume maps required to expose needed files.
        volume_maps = {
            PYTHON_RUNTIME_DIRNAME: PYTHON_RUNTIME_VOLUME,
            PYTHON_PROGRAM_FILENAME: PYTHON_PROGRAM_VOLUME,
            PYTHON_INPUTS_FILENAME: PYTHON_INPUTS_VOLUME,
        }

        return volume_maps

Response: Prepare the runtime directory for the current process.