Instruction:
Can you write a function in Python 3 where it
prepares the private execution directory for the given data object.

Response:
def _prepare_data_dir(self, data):
        """Prepare destination directory where the data will live.

        :param data: The :class:`~resolwe.flow.models.Data` object for
            which to prepare the private execution directory.
        :return: The prepared data directory path.
        :rtype: str
        """
        logger.debug(__("Preparing data directory for Data with id {}.", data.id))

        with transaction.atomic():
            # Create a temporary random location and then override it with data
            # location id since object has to be created first.
            # TODO Find a better solution, e.g. defer the database constraint.
            temporary_location_string = uuid.uuid4().hex[:10]
            data_location = DataLocation.objects.create(subpath=temporary_location_string)
            data_location.subpath = str(data_location.id)
            data_location.save()
            data_location.data.add(data)

        output_path = self._get_per_data_dir('DATA_DIR', data_location.subpath)
        dir_mode = self.settings_actual.get('FLOW_EXECUTOR', {}).get('DATA_DIR_MODE', 0o755)
        os.mkdir(output_path, mode=dir_mode)
        # os.mkdir is not guaranteed to set the given mode
        os.chmod(output_path, dir_mode)
        return output_path


Instruction:
Write a Python 3 function that can
prepare the context for the executor.

Response:
def _prepare_context(self, data_id, data_dir, runtime_dir, **kwargs):
        """Prepare settings and constants JSONs for the executor.

        Settings and constants provided by other ``resolwe`` modules and
        :class:`~django.conf.settings` are all inaccessible in the
        executor once it is deployed, so they need to be serialized into
        the runtime directory.

        :param data_id: The :class:`~resolwe.flow.models.Data` object id
            being prepared for.
        :param data_dir: The target execution directory for this
            :class:`~resolwe.flow.models.Data` object.
        :param runtime_dir: The target runtime support directory for
            this :class:`~resolwe.flow.models.Data` object; this is
            where the environment is serialized into.
        :param kwargs: Extra settings to include in the main settings
            file.
        """
        files = {}
        secrets = {}

        settings_dict = {}
        settings_dict['DATA_DIR'] = data_dir
        settings_dict['REDIS_CHANNEL_PAIR'] = state.MANAGER_EXECUTOR_CHANNELS
        files[ExecutorFiles.EXECUTOR_SETTINGS] = settings_dict

        django_settings = {}
        django_settings.update(self.settings_actual)
        django_settings.update(kwargs)
        files[ExecutorFiles.DJANGO_SETTINGS] = django_settings

        # Add scheduling classes.
        files[ExecutorFiles.PROCESS_META] = {
            k: getattr(Process, k) for k in dir(Process)
            if k.startswith('SCHEDULING_CLASS_') and isinstance(getattr(Process, k), str)
        }

        # Add Data status constants.
        files[ExecutorFiles.DATA_META] = {
            k: getattr(Data, k) for k in dir(Data)
            if k.startswith('STATUS_') and isinstance(getattr(Data, k), str)
        }

        # Extend the settings with whatever the executor wants.
        self.executor.extend_settings(data_id, files, secrets)

        # Save the settings into the various files in the runtime dir.
        settings_dict[ExecutorFiles.FILE_LIST_KEY] = list(files.keys())
        for file_name in files:
            file_path = os.path.join(runtime_dir, file_name)
            with open(file_path, 'wt') as json_file:
                json.dump(files[file_name], json_file, cls=SettingsJSONifier)

        # Save the secrets in the runtime dir, with permissions to prevent listing the given
        # directory.
        secrets_dir = os.path.join(runtime_dir, ExecutorFiles.SECRETS_DIR)
        os.makedirs(secrets_dir, mode=0o300)
        for file_name, value in secrets.items():
            file_path = os.path.join(secrets_dir, file_name)

            # Set umask to 0 to ensure that we set the correct permissions.
            old_umask = os.umask(0)
            try:
                # We need to use os.open in order to correctly enforce file creation. Otherwise,
                # there is a race condition which can be used to create the file with different
                # ownership/permissions.
                file_descriptor = os.open(file_path, os.O_WRONLY | os.O_CREAT, mode=0o600)
                with os.fdopen(file_descriptor, 'w') as raw_file:
                    raw_file.write(value)
            finally:
                os.umask(old_umask)


Instruction:
Make a summary of the following Python 3 code
def _prepare_executor(self, data, executor):
        logger.debug(__("Preparing executor for Data with id {}", data.id))

        # Both of these imports are here only to get the packages' paths.
        import resolwe.flow.executors as executor_package

        exec_dir = os.path.dirname(inspect.getsourcefile(executor_package))
        dest_dir = self._get_per_data_dir('RUNTIME_DIR', data.location.subpath)
        dest_package_dir = os.path.join(dest_dir, 'executors')
        shutil.copytree(exec_dir, dest_package_dir)
        dir_mode = self.settings_actual.get('FLOW_EXECUTOR', {}).get('RUNTIME_DIR_MODE', 0o755)
        os.chmod(dest_dir, dir_mode)

        class_name = executor.rpartition('.executors.')[-1]
        return '.{}'.format(class_name), dest_dir

Response: Prepare the executor class for the given data object.


Instruction:
Can you implement a function in Python 3 that
copies the script into the destination directory.

Response:
def _prepare_script(self, dest_dir, program):
        """Copy the script into the destination directory.

        :param dest_dir: The target directory where the script will be
            saved.
        :param program: The script text to be saved.
        :return: The name of the script file.
        :rtype: str
        """
        script_name = ExecutorFiles.PROCESS_SCRIPT
        dest_file = os.path.join(dest_dir, script_name)
        with open(dest_file, 'wt') as dest_file_obj:
            dest_file_obj.write(program)
        os.chmod(dest_file, 0o700)
        return script_name


Instruction:
How would you explain what the following Python 3 function does
async def handle_control_event(self, message):
        cmd = message[WorkerProtocol.COMMAND]
        logger.debug(__("Manager worker got channel command '{}'.", cmd))

        # Prepare settings for use; Django overlaid by state overlaid by
        # anything immediate in the current packet.
        immediates = {}
        if cmd == WorkerProtocol.COMMUNICATE:
            immediates = message.get(WorkerProtocol.COMMUNICATE_SETTINGS, {}) or {}
        override = self.state.settings_override or {}
        override.update(immediates)
        self.settings_actual = self._marshal_settings()
        self.settings_actual.update(override)

        if cmd == WorkerProtocol.COMMUNICATE:
            try:
                await database_sync_to_async(self._data_scan)(**message[WorkerProtocol.COMMUNICATE_EXTRA])
            except Exception:
                logger.exception("Unknown error occured while processing communicate control command.")
                raise
            finally:
                await self.sync_counter.dec('communicate')

        elif cmd == WorkerProtocol.FINISH:
            try:
                data_id = message[WorkerProtocol.DATA_ID]
                data_location = DataLocation.objects.get(data__id=data_id)
                if not getattr(settings, 'FLOW_MANAGER_KEEP_DATA', False):
                    try:
                        def handle_error(func, path, exc_info):
                            """Handle permission errors while removing data directories."""
                            if isinstance(exc_info[1], PermissionError):
                                os.chmod(path, 0o700)
                                shutil.rmtree(path)

                        # Remove secrets directory, but leave the rest of the runtime directory
                        # intact. Runtime directory will be removed during data purge, when the
                        # data object is removed.
                        secrets_dir = os.path.join(
                            self._get_per_data_dir('RUNTIME_DIR', data_location.subpath),
                            ExecutorFiles.SECRETS_DIR
                        )
                        shutil.rmtree(secrets_dir, onerror=handle_error)
                    except OSError:
                        logger.exception("Manager exception while removing data runtime directory.")

                if message[WorkerProtocol.FINISH_SPAWNED]:
                    await database_sync_to_async(self._data_scan)(**message[WorkerProtocol.FINISH_COMMUNICATE_EXTRA])
            except Exception:
                logger.exception(
                    "Unknown error occured while processing finish control command.",
                    extra={'data_id': data_id}
                )
                raise
            finally:
                await self.sync_counter.dec('executor')

        elif cmd == WorkerProtocol.ABORT:
            await self.sync_counter.dec('executor')

        else:
            logger.error(__("Ignoring unknown manager control command '{}'.", cmd))

Response: Handle an event from the Channels layer callback do not call directly.


Instruction:
Make a summary of the following Python 3 code
def _ensure_counter(self):
        if not isinstance(self.sync_counter, self._SynchronizationManager):
            self.sync_counter = self._SynchronizationManager()

Response: Ensure the sync counter is a valid non - dummy object.


Instruction:
Explain what the following Python 3 code does
async def execution_barrier(self):
        async def _barrier():
            """Enter the sync block and exit the app afterwards."""
            async with self.sync_counter:
                pass
            await consumer.exit_consumer()

        self._ensure_counter()
        await asyncio.wait([
            _barrier(),
            consumer.run_consumer(),
        ])
        self.sync_counter = self._SynchronizationManagerDummy()

Response: Wait for all executors to finish.


Instruction:
Make a summary of the following Python 3 code
async def communicate(self, data_id=None, run_sync=False, save_settings=True):
        executor = getattr(settings, 'FLOW_EXECUTOR', {}).get('NAME', 'resolwe.flow.executors.local')
        logger.debug(__(
            "Manager sending communicate command on '{}' triggered by Data with id {}.",
            state.MANAGER_CONTROL_CHANNEL,
            data_id,
        ))

        saved_settings = self.state.settings_override
        if save_settings:
            saved_settings = self._marshal_settings()
            self.state.settings_override = saved_settings

        if run_sync:
            self._ensure_counter()
        await self.sync_counter.inc('communicate')
        try:
            await consumer.send_event({
                WorkerProtocol.COMMAND: WorkerProtocol.COMMUNICATE,
                WorkerProtocol.COMMUNICATE_SETTINGS: saved_settings,
                WorkerProtocol.COMMUNICATE_EXTRA: {
                    'data_id': data_id,
                    'executor': executor,
                },
            })
        except ChannelFull:
            logger.exception("ChannelFull error occurred while sending communicate message.")
            await self.sync_counter.dec('communicate')

        if run_sync and not self.sync_counter.active:
            logger.debug(__(
                "Manager on channel '{}' entering synchronization block.",
                state.MANAGER_CONTROL_CHANNEL
            ))
            await self.execution_barrier()
            logger.debug(__(
                "Manager on channel '{}' exiting synchronization block.",
                state.MANAGER_CONTROL_CHANNEL
            ))

Response: This method is used to send a COMMUNICATE command to the manager s channel workers.


Instruction:
Here you have a function in Python 3, explain what it does
def _data_execute(self, data, program, executor):
        if not program:
            return

        logger.debug(__("Manager preparing Data with id {} for processing.", data.id))

        # Prepare the executor's environment.
        try:
            executor_env_vars = self.get_executor().get_environment_variables()
            program = self._include_environment_variables(program, executor_env_vars)
            data_dir = self._prepare_data_dir(data)
            executor_module, runtime_dir = self._prepare_executor(data, executor)

            # Execute execution engine specific runtime preparation.
            execution_engine = data.process.run.get('language', None)
            volume_maps = self.get_execution_engine(execution_engine).prepare_runtime(runtime_dir, data)

            self._prepare_context(data.id, data_dir, runtime_dir, RUNTIME_VOLUME_MAPS=volume_maps)
            self._prepare_script(runtime_dir, program)

            argv = [
                '/bin/bash',
                '-c',
                self.settings_actual.get('FLOW_EXECUTOR', {}).get('PYTHON', '/usr/bin/env python')
                + ' -m executors ' + executor_module
            ]
        except PermissionDenied as error:
            data.status = Data.STATUS_ERROR
            data.process_error.append("Permission denied for process: {}".format(error))
            data.save()
            return
        except OSError as err:
            logger.error(__(
                "OSError occurred while preparing data {} (will skip): {}",
                data.id, err
            ))
            return

        # Hand off to the run() method for execution.
        logger.info(__("Running {}", runtime_dir))
        self.run(data, runtime_dir, argv)

Response: Execute the data object.


Instruction:
Can you create a Python 3 function that
scans for new Data objects and execute them.

Response:
def _data_scan(self, data_id=None, executor='resolwe.flow.executors.local', **kwargs):
        """Scan for new Data objects and execute them.

        :param data_id: Optional id of Data object which (+ its
            children) should be scanned. If it is not given, all
            resolving objects are processed.
        :param executor: The fully qualified name of the executor to use
            for all :class:`~resolwe.flow.models.Data` objects
            discovered in this pass.
        """
        def process_data_object(data):
            """Process a single data object."""
            # Lock for update. Note that we want this transaction to be as short as possible in
            # order to reduce contention and avoid deadlocks. This is why we do not lock all
            # resolving objects for update, but instead only lock one object at a time. This
            # allows managers running in parallel to process different objects.
            data = Data.objects.select_for_update().get(pk=data.pk)
            if data.status != Data.STATUS_RESOLVING:
                # The object might have already been processed while waiting for the lock to be
                # obtained. In this case, skip the object.
                return

            dep_status = dependency_status(data)

            if dep_status == Data.STATUS_ERROR:
                data.status = Data.STATUS_ERROR
                data.process_error.append("One or more inputs have status ERROR")
                data.process_rc = 1
                data.save()
                return

            elif dep_status != Data.STATUS_DONE:
                return

            if data.process.run:
                try:
                    execution_engine = data.process.run.get('language', None)
                    # Evaluation by the execution engine may spawn additional data objects and
                    # perform other queries on the database. Queries of all possible execution
                    # engines need to be audited for possibilities of deadlocks in case any
                    # additional locks are introduced. Currently, we only take an explicit lock on
                    # the currently processing object.
                    program = self.get_execution_engine(execution_engine).evaluate(data)
                except (ExecutionError, InvalidEngineError) as error:
                    data.status = Data.STATUS_ERROR
                    data.process_error.append("Error in process script: {}".format(error))
                    data.save()
                    return

                # Set allocated resources:
                resource_limits = data.process.get_resource_limits()
                data.process_memory = resource_limits['memory']
                data.process_cores = resource_limits['cores']
            else:
                # If there is no run section, then we should not try to run anything. But the
                # program must not be set to None as then the process will be stuck in waiting
                # state.
                program = ''

            if data.status != Data.STATUS_DONE:
                # The data object may already be marked as done by the execution engine. In this
                # case we must not revert the status to STATUS_WAITING.
                data.status = Data.STATUS_WAITING
            data.save(render_name=True)

            # Actually run the object only if there was nothing with the transaction.
            transaction.on_commit(
                # Make sure the closure gets the right values here, since they're
                # changed in the loop.
                lambda d=data, p=program: self._data_execute(d, p, executor)
            )

        logger.debug(__("Manager processing communicate command triggered by Data with id {}.", data_id))

        if is_testing():
            # NOTE: This is a work-around for Django issue #10827
            # (https://code.djangoproject.com/ticket/10827), same as in
            # TestCaseHelpers._pre_setup(). Because the worker is running
            # independently, it must clear the cache on its own.
            ContentType.objects.clear_cache()

            # Ensure settings overrides apply
            self.discover_engines(executor=executor)

        try:
            queryset = Data.objects.filter(status=Data.STATUS_RESOLVING)
            if data_id is not None:
                # Scan only given data object and its children.
                queryset = queryset.filter(Q(parents=data_id) | Q(id=data_id)).distinct()

            for data in queryset:
                try:
                    with transaction.atomic():
                        process_data_object(data)

                        # All data objects created by the execution engine are commited after this
                        # point and may be processed by other managers running in parallel. At the
                        # same time, the lock for the current data object is released.
                except Exception as error:  # pylint: disable=broad-except
                    logger.exception(__(
                        "Unhandled exception in _data_scan while processing data object {}.",
                        data.pk
                    ))

                    # Unhandled error while processing a data object. We must set its
                    # status to STATUS_ERROR to prevent the object from being retried
                    # on next _data_scan run. We must perform this operation without
                    # using the Django ORM as using the ORM may be the reason the error
                    # occurred in the first place.
                    error_msg = "Internal error: {}".format(error)
                    process_error_field = Data._meta.get_field('process_error')  # pylint: disable=protected-access
                    max_length = process_error_field.base_field.max_length
                    if len(error_msg) > max_length:
                        error_msg = error_msg[:max_length - 3] + '...'

                    try:
                        with connection.cursor() as cursor:
                            cursor.execute(
                                """
                                    UPDATE {table}
                                    SET
                                        status = %(status)s,
                                        process_error = process_error || (%(error)s)::varchar[]
                                    WHERE id = %(id)s
                                """.format(
                                    table=Data._meta.db_table  # pylint: disable=protected-access
                                ),
                                {
                                    'status': Data.STATUS_ERROR,
                                    'error': [error_msg],
                                    'id': data.pk
                                }
                            )
                    except Exception as error:  # pylint: disable=broad-except
                        # If object's state cannot be changed due to some database-related
                        # issue, at least skip the object for this run.
                        logger.exception(__(
                            "Unhandled exception in _data_scan while trying to emit error for {}.",
                            data.pk
                        ))

        except IntegrityError as exp:
            logger.error(__("IntegrityError in manager {}", exp))
            return


Instruction:
Create a Python 3 function to
return an expression engine instance.

Response:
def get_expression_engine(self, name):
        """Return an expression engine instance."""
        try:
            return self.expression_engines[name]
        except KeyError:
            raise InvalidEngineError("Unsupported expression engine: {}".format(name))


Instruction:
Can you write a function in Python 3 where it
returns an execution engine instance.

Response:
def get_execution_engine(self, name):
        """Return an execution engine instance."""
        try:
            return self.execution_engines[name]
        except KeyError:
            raise InvalidEngineError("Unsupported execution engine: {}".format(name))


Instruction:
Write a Python 3 script to
extract subjects from a DataONE PEM encoded X. 509 certificate.

Response:
def extract_subjects(cert_pem):
    """Extract subjects from a DataONE PEM (Base64) encoded X.509 v3 certificate.

    Args:
      cert_pem: str or bytes
        PEM (Base64) encoded X.509 v3 certificate

    Returns:
      2-tuple:
        - The primary subject string, extracted from the certificate DN.
        - A set of equivalent identities, group memberships and inferred symbolic
          subjects extracted from the SubjectInfo (if present.)
        - All returned subjects are DataONE compliant serializations.
        - A copy of the primary subject is always included in the set of equivalent
          identities.

    """
    primary_str, subject_info_xml = d1_common.cert.x509.extract_subjects(cert_pem)
    equivalent_set = {
        primary_str,
        d1_common.const.SUBJECT_AUTHENTICATED,
        d1_common.const.SUBJECT_PUBLIC,
    }
    if subject_info_xml is not None:
        equivalent_set |= d1_common.cert.subject_info.extract_subjects(
            subject_info_xml, primary_str
        )
    return primary_str, equivalent_set


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _get_collection_for_user(self, collection_id, user):
        collection_query = Collection.objects.filter(pk=collection_id)
        if not collection_query.exists():
            raise exceptions.ValidationError('Collection id does not exist')

        collection = collection_query.first()
        if not user.has_perm('add_collection', obj=collection):
            if user.is_authenticated:
                raise exceptions.PermissionDenied()
            else:
                raise exceptions.NotFound()

        return collection

Response: Check that collection exists and user has add permission.


Instruction:
How would you explain what the following Python 3 function does
def _get_entities(self, user, ids):
        queryset = get_objects_for_user(user, 'view_entity', Entity.objects.filter(id__in=ids))
        actual_ids = queryset.values_list('id', flat=True)
        missing_ids = list(set(ids) - set(actual_ids))
        if missing_ids:
            raise exceptions.ParseError(
                "Entities with the following ids not found: {}" .format(', '.join(map(str, missing_ids)))
            )

        return queryset

Response: Return entities queryset based on provided entity ids.


Instruction:
Write a Python 3 function that can
apply permissions to data objects in Entity.

Response:
def set_content_permissions(self, user, obj, payload):
        """Apply permissions to data objects in ``Entity``."""
        # Data doesn't have "ADD" permission, so it has to be removed
        payload = remove_permission(payload, 'add')

        for data in obj.data.all():
            if user.has_perm('share_data', data):
                update_permission(data, payload)


Instruction:
Create a Python 3 function for
destroying a model instance.

Response:
def destroy(self, request, *args, **kwargs):
        """Destroy a model instance.

        If ``delete_content`` flag is set in query parameters, also all
        Data objects contained in entity will be deleted.
        """
        obj = self.get_object()
        user = request.user

        if strtobool(request.query_params.get('delete_content', 'false')):
            for data in obj.data.all():
                if user.has_perm('edit_data', data):
                    data.delete()

            # If all data objects in an entity are removed, the entity may
            # have already been removed, so there is no need to call destroy.
            if not Entity.objects.filter(pk=obj.pk).exists():
                return Response(status=status.HTTP_204_NO_CONTENT)

        # NOTE: Collection's ``destroy`` method should be skiped, so we
        # intentionaly call it's parent.
        return super(CollectionViewSet, self).destroy(  # pylint: disable=no-member,bad-super-call
            request, *args, **kwargs
        )


Instruction:
Write a Python 3 script to
add Entity to a collection.

Response:
def add_to_collection(self, request, pk=None):
        """Add Entity to a collection."""
        entity = self.get_object()

        # TODO use `self.get_ids` (and elsewhere). Backwards
        # incompatible because raised error's response contains
        # ``detail`` instead of ``error``).
        if 'ids' not in request.data:
            return Response({"error": "`ids` parameter is required"}, status=status.HTTP_400_BAD_REQUEST)

        for collection_id in request.data['ids']:
            self._get_collection_for_user(collection_id, request.user)

        for collection_id in request.data['ids']:
            entity.collections.add(collection_id)

            collection = Collection.objects.get(pk=collection_id)
            for data in entity.data.all():
                collection.data.add(data)

        return Response()


Instruction:
How would you implement a function in Python 3 that
adds data to Entity and its collection.

Response:
def add_data(self, request, pk=None):
        """Add data to Entity and it's collection."""
        # add data to entity
        resp = super().add_data(request, pk)

        # add data to collections in which entity is
        entity = self.get_object()
        for collection in entity.collections.all():
            collection.data.add(*request.data['ids'])

        return resp


Instruction:
Can you create a Python 3 function that
moves samples from source to destination collection.

Response:
def move_to_collection(self, request, *args, **kwargs):
        """Move samples from source to destination collection."""
        ids = self.get_ids(request.data)
        src_collection_id = self.get_id(request.data, 'source_collection')
        dst_collection_id = self.get_id(request.data, 'destination_collection')

        src_collection = self._get_collection_for_user(src_collection_id, request.user)
        dst_collection = self._get_collection_for_user(dst_collection_id, request.user)

        entity_qs = self._get_entities(request.user, ids)
        entity_qs.move_to_collection(src_collection, dst_collection)

        return Response()


Instruction:
Create a Python 3 function to
update an entity. Original queryset produces a temporary database table whose rows cannot be selected for an update. As a workaround, we patch get_queryset function to return only Entity objects without additional data that is not needed for the update.

Response:
def update(self, request, *args, **kwargs):
        """Update an entity.

        Original queryset produces a temporary database table whose rows
        cannot be selected for an update. As a workaround, we patch
        get_queryset function to return only Entity objects without
        additional data that is not needed for the update.
        """
        orig_get_queryset = self.get_queryset

        def patched_get_queryset():
            """Patched get_queryset method."""
            entity_ids = orig_get_queryset().values_list('id', flat=True)
            return Entity.objects.filter(id__in=entity_ids)

        self.get_queryset = patched_get_queryset
        resp = super().update(request, *args, **kwargs)
        self.get_queryset = orig_get_queryset
        return resp


Instruction:
Implement a Python 3 function for
starting the process execution.

Response:
async def start(self):
        """Start process execution."""
        # arguments passed to the Docker command
        command_args = {
            'command': self.command,
            'container_image': self.requirements.get('image', constants.DEFAULT_CONTAINER_IMAGE),
        }

        # Get limit defaults.
        limit_defaults = SETTINGS.get('FLOW_PROCESS_RESOURCE_DEFAULTS', {})

        # Set resource limits.
        limits = []
        # Each core is equivalent to 1024 CPU shares. The default for Docker containers
        # is 1024 shares (we don't need to explicitly set that).
        limits.append('--cpu-shares={}'.format(int(self.process['resource_limits']['cores']) * 1024))

        # Some SWAP is needed to avoid OOM signal. Swappiness is low to prevent
        # extensive usage of SWAP (this would reduce the performance).
        memory = self.process['resource_limits']['memory'] + DOCKER_MEMORY_HARD_LIMIT_BUFFER
        memory_swap = int(memory * DOCKER_MEMORY_SWAP_RATIO)

        limits.append('--memory={}m'.format(memory))
        limits.append('--memory-swap={}m'.format(memory_swap))
        limits.append('--memory-reservation={}m'.format(self.process['resource_limits']['memory']))
        limits.append('--memory-swappiness={}'.format(DOCKER_MEMORY_SWAPPINESS))

        # Set ulimits for interactive processes to prevent them from running too long.
        if self.process['scheduling_class'] == PROCESS_META['SCHEDULING_CLASS_INTERACTIVE']:
            # TODO: This is not very good as each child gets the same limit.
            limits.append('--ulimit cpu={}'.format(limit_defaults.get('cpu_time_interactive', 30)))

        command_args['limits'] = ' '.join(limits)

        # set container name
        self.container_name_prefix = SETTINGS.get('FLOW_EXECUTOR', {}).get('CONTAINER_NAME_PREFIX', 'resolwe')
        command_args['container_name'] = '--name={}'.format(self._generate_container_name())

        if 'network' in self.resources:
            # Configure Docker network mode for the container (if specified).
            # By default, current Docker versions use the 'bridge' mode which
            # creates a network stack on the default Docker bridge.
            network = SETTINGS.get('FLOW_EXECUTOR', {}).get('NETWORK', '')
            command_args['network'] = '--net={}'.format(network) if network else ''
        else:
            # No network if not specified.
            command_args['network'] = '--net=none'

        # Security options.
        security = []

        # Generate and set seccomp policy to limit syscalls.
        policy_file = tempfile.NamedTemporaryFile(mode='w')
        json.dump(SECCOMP_POLICY, policy_file)
        policy_file.file.flush()
        if not SETTINGS.get('FLOW_DOCKER_DISABLE_SECCOMP', False):
            security.append('--security-opt seccomp={}'.format(policy_file.name))
        self.temporary_files.append(policy_file)

        # Drop all capabilities and only add ones that are needed.
        security.append('--cap-drop=all')

        command_args['security'] = ' '.join(security)

        # Setup Docker volumes.
        def new_volume(kind, base_dir_name, volume, path=None, read_only=True):
            """Generate a new volume entry.

            :param kind: Kind of volume, which is used for getting extra options from
                settings (the ``FLOW_DOCKER_VOLUME_EXTRA_OPTIONS`` setting)
            :param base_dir_name: Name of base directory setting for volume source path
            :param volume: Destination volume mount point
            :param path: Optional additional path atoms appended to source path
            :param read_only: True to make the volume read-only
            """
            if path is None:
                path = []

            path = [str(atom) for atom in path]

            options = set(SETTINGS.get('FLOW_DOCKER_VOLUME_EXTRA_OPTIONS', {}).get(kind, '').split(','))
            options.discard('')
            # Do not allow modification of read-only option.
            options.discard('ro')
            options.discard('rw')

            if read_only:
                options.add('ro')
            else:
                options.add('rw')

            return {
                'src': os.path.join(SETTINGS['FLOW_EXECUTOR'].get(base_dir_name, ''), *path),
                'dest': volume,
                'options': ','.join(options),
            }

        volumes = [
            new_volume(
                'data', 'DATA_DIR', constants.DATA_VOLUME, [DATA_LOCATION['subpath']], read_only=False
            ),
            new_volume('data_all', 'DATA_DIR', constants.DATA_ALL_VOLUME),
            new_volume('upload', 'UPLOAD_DIR', constants.UPLOAD_VOLUME, read_only=False),
            new_volume(
                'secrets',
                'RUNTIME_DIR',
                constants.SECRETS_VOLUME,
                [DATA_LOCATION['subpath'], ExecutorFiles.SECRETS_DIR]
            ),
        ]

        # Generate dummy passwd and create mappings for it. This is required because some tools
        # inside the container may try to lookup the given UID/GID and will crash if they don't
        # exist. So we create minimal user/group files.
        passwd_file = tempfile.NamedTemporaryFile(mode='w')
        passwd_file.write('root:x:0:0:root:/root:/bin/bash\n')
        passwd_file.write('user:x:{}:{}:user:/:/bin/bash\n'.format(os.getuid(), os.getgid()))
        passwd_file.file.flush()
        self.temporary_files.append(passwd_file)

        group_file = tempfile.NamedTemporaryFile(mode='w')
        group_file.write('root:x:0:\n')
        group_file.write('user:x:{}:user\n'.format(os.getgid()))
        group_file.file.flush()
        self.temporary_files.append(group_file)

        volumes += [
            new_volume('users', None, '/etc/passwd', [passwd_file.name]),
            new_volume('users', None, '/etc/group', [group_file.name]),
        ]

        # Create volumes for tools.
        # NOTE: To prevent processes tampering with tools, all tools are mounted read-only
        self.tools_volumes = []
        for index, tool in enumerate(self.get_tools_paths()):
            self.tools_volumes.append(new_volume(
                'tools',
                None,
                os.path.join('/usr/local/bin/resolwe', str(index)),
                [tool]
            ))

        volumes += self.tools_volumes

        # Create volumes for runtime (all read-only).
        runtime_volume_maps = SETTINGS.get('RUNTIME_VOLUME_MAPS', None)
        if runtime_volume_maps:
            for src, dst in runtime_volume_maps.items():
                volumes.append(new_volume(
                    'runtime',
                    'RUNTIME_DIR',
                    dst,
                    [DATA_LOCATION['subpath'], src],
                ))

        # Add any extra volumes verbatim.
        volumes += SETTINGS.get('FLOW_DOCKER_EXTRA_VOLUMES', [])

        # Make sure that tmp dir exists.
        os.makedirs(constants.TMPDIR, mode=0o755, exist_ok=True)

        # Create Docker --volume parameters from volumes.
        command_args['volumes'] = ' '.join(['--volume="{src}":"{dest}":{options}'.format(**volume)
                                            for volume in volumes])

        # Set working directory to the data volume.
        command_args['workdir'] = '--workdir={}'.format(constants.DATA_VOLUME)

        # Change user inside the container.
        command_args['user'] = '--user={}:{}'.format(os.getuid(), os.getgid())

        # A non-login Bash shell should be used here (a subshell will be spawned later).
        command_args['shell'] = '/bin/bash'

        # Check if image exists locally. If not, command will exit with non-zero returncode
        check_command = '{command} image inspect {container_image}'.format(**command_args)

        logger.debug("Checking existence of docker image: {}".format(command_args['container_image']))

        check_proc = await subprocess.create_subprocess_exec(  # pylint: disable=no-member
            *shlex.split(check_command),
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE
        )
        await check_proc.communicate()

        if check_proc.returncode != 0:
            pull_command = '{command} pull {container_image}'.format(**command_args)

            logger.info("Pulling docker image: {}".format(command_args['container_image']))

            pull_proc = await subprocess.create_subprocess_exec(  # pylint: disable=no-member
                *shlex.split(pull_command),
                stdout=subprocess.PIPE,
                stderr=subprocess.PIPE
            )
            _, stderr = await pull_proc.communicate()

            if pull_proc.returncode != 0:
                error_msg = "Docker failed to pull {} image.".format(command_args['container_image'])
                if stderr:
                    error_msg = '\n'.join([error_msg, stderr.decode('utf-8')])
                raise RuntimeError(error_msg)

        docker_command = (
            '{command} run --rm --interactive {container_name} {network} {volumes} {limits} '
            '{security} {workdir} {user} {container_image} {shell}'.format(**command_args)
        )

        logger.info("Starting docker container with command: {}".format(docker_command))
        start_time = time.time()

        # Workaround for pylint issue #1469
        # (https://github.com/PyCQA/pylint/issues/1469).
        self.proc = await subprocess.create_subprocess_exec(  # pylint: disable=no-member
            *shlex.split(docker_command),
            limit=4 * (2 ** 20),  # 4MB buffer size for line buffering
            stdin=subprocess.PIPE,
            stdout=subprocess.PIPE,
            stderr=subprocess.STDOUT
        )

        stdout = []

        async def wait_for_container():
            """Wait for Docker container to start to avoid blocking the code that uses it."""
            self.proc.stdin.write(('echo PING' + os.linesep).encode('utf-8'))
            await self.proc.stdin.drain()
            while True:
                line = await self.proc.stdout.readline()
                stdout.append(line)
                if line.rstrip() == b'PING':
                    break
                if self.proc.stdout.at_eof():
                    raise RuntimeError()

        try:
            await asyncio.wait_for(wait_for_container(), timeout=DOCKER_START_TIMEOUT)
        except (asyncio.TimeoutError, RuntimeError):
            error_msg = "Docker container has not started for {} seconds.".format(DOCKER_START_TIMEOUT)
            stdout = ''.join([line.decode('utf-8') for line in stdout if line])
            if stdout:
                error_msg = '\n'.join([error_msg, stdout])
            raise RuntimeError(error_msg)

        end_time = time.time()
        logger.info("It took {:.2f}s for Docker container to start".format(end_time - start_time))

        self.stdout = self.proc.stdout


Instruction:
Here you have a function in Python 3, explain what it does
async def run_script(self, script):
        # Create a Bash command to add all the tools to PATH.
        tools_paths = ':'.join([map_["dest"] for map_ in self.tools_volumes])
        add_tools_path = 'export PATH=$PATH:{}'.format(tools_paths)
        # Spawn another child bash, to avoid running anything as PID 1, which has special
        # signal handling (e.g., cannot be SIGKILL-ed from inside).
        # A login Bash shell is needed to source /etc/profile.
        bash_line = '/bin/bash --login; exit $?' + os.linesep
        script = os.linesep.join(['set -x', 'set +B', add_tools_path, script]) + os.linesep
        self.proc.stdin.write(bash_line.encode('utf-8'))
        await self.proc.stdin.drain()
        self.proc.stdin.write(script.encode('utf-8'))
        await self.proc.stdin.drain()
        self.proc.stdin.close()

Response: Execute the script and save results.


Instruction:
Write a Python 3 function that can
terminate a running script.

Response:
async def terminate(self):
        """Terminate a running script."""
        # Workaround for pylint issue #1469
        # (https://github.com/PyCQA/pylint/issues/1469).
        cmd = await subprocess.create_subprocess_exec(  # pylint: disable=no-member
            *shlex.split('{} rm -f {}'.format(self.command, self._generate_container_name()))
        )
        await cmd.wait()
        await self.proc.wait()

        await super().terminate()


Instruction:
Write a Python 3 script to
send an update to manager and terminate the process if it fails.

Response:
async def _send_manager_command(self, *args, **kwargs):
        """Send an update to manager and terminate the process if it fails."""
        resp = await send_manager_command(*args, **kwargs)

        if resp is False:
            await self.terminate()


Instruction:
Implement a function in Python 3 to
update the status of the current data object.

Response:
async def update_data_status(self, **kwargs):
        """Update (PATCH) Data object.

        :param kwargs: The dictionary of
            :class:`~resolwe.flow.models.Data` attributes to be changed.
        """
        await self._send_manager_command(ExecutorProtocol.UPDATE, extra_fields={
            ExecutorProtocol.UPDATE_CHANGESET: kwargs
        })


Instruction:
Can you write a function in Python 3 where it
executes the script and save the results.

Response:
async def run(self, data_id, script):
        """Execute the script and save results."""
        logger.debug("Executor for Data with id {} has started.".format(data_id))
        try:
            finish_fields = await self._run(data_id, script)
        except SystemExit as ex:
            raise ex
        except Exception as error:  # pylint: disable=broad-except
            logger.exception("Unhandled exception in executor")

            # Send error report.
            await self.update_data_status(process_error=[str(error)], status=DATA_META['STATUS_ERROR'])

            finish_fields = {
                ExecutorProtocol.FINISH_PROCESS_RC: 1,
            }

        if finish_fields is not None:
            await self._send_manager_command(ExecutorProtocol.FINISH, extra_fields=finish_fields)


Instruction:
Create a Python 3 function to
ensure a new file is created and opened for writing.

Response:
def _create_file(self, filename):
        """Ensure a new file is created and opened for writing."""
        file_descriptor = os.open(filename, os.O_WRONLY | os.O_CREAT | os.O_EXCL)
        return os.fdopen(file_descriptor, 'w')


Instruction:
Can you generate a brief explanation for the following Python 3 code
async def _run(self, data_id, script):
        self.data_id = data_id

        # Fetch data instance to get any executor requirements.
        self.process = PROCESS
        requirements = self.process['requirements']
        self.requirements = requirements.get('executor', {}).get(self.name, {})  # pylint: disable=no-member
        self.resources = requirements.get('resources', {})

        logger.debug("Preparing output files for Data with id {}".format(data_id))
        os.chdir(EXECUTOR_SETTINGS['DATA_DIR'])
        try:
            log_file = self._create_file('stdout.txt')
            json_file = self._create_file('jsonout.txt')
        except FileExistsError:
            logger.error("Stdout or jsonout out file already exists.")
            # Looks like executor was already ran for this Data object,
            # so don't raise the error to prevent setting status to error.
            await self._send_manager_command(ExecutorProtocol.ABORT, expect_reply=False)
            return

        proc_pid = await self.start()

        await self.update_data_status(
            status=DATA_META['STATUS_PROCESSING'],
            process_pid=proc_pid
        )

        # Run process and handle intermediate results
        logger.info("Running program for Data with id {}".format(data_id))
        logger.debug("The program for Data with id {} is: \n{}".format(data_id, script))
        await self.run_script(script)
        spawn_processes = []
        output = {}
        process_error, process_warning, process_info = [], [], []
        process_progress, process_rc = 0, 0

        # read process output
        try:
            stdout = self.get_stdout()
            while True:
                line = await stdout.readline()
                logger.debug("Process's output: {}".format(line.strip()))

                if not line:
                    break
                line = line.decode('utf-8')

                try:
                    if line.strip().startswith('run'):
                        # Save process and spawn if no errors
                        log_file.write(line)
                        log_file.flush()

                        for obj in iterjson(line[3:].strip()):
                            spawn_processes.append(obj)
                    elif line.strip().startswith('export'):
                        file_name = line[6:].strip()

                        export_folder = SETTINGS['FLOW_EXECUTOR']['UPLOAD_DIR']
                        unique_name = 'export_{}'.format(uuid.uuid4().hex)
                        export_path = os.path.join(export_folder, unique_name)

                        self.exported_files_mapper[self.data_id][file_name] = unique_name

                        shutil.move(file_name, export_path)
                    else:
                        # If JSON, save to MongoDB
                        updates = {}
                        for obj in iterjson(line):
                            for key, val in obj.items():
                                if key.startswith('proc.'):
                                    if key == 'proc.error':
                                        process_error.append(val)
                                        if not process_rc:
                                            process_rc = 1
                                            updates['process_rc'] = process_rc
                                        updates['process_error'] = process_error
                                        updates['status'] = DATA_META['STATUS_ERROR']
                                    elif key == 'proc.warning':
                                        process_warning.append(val)
                                        updates['process_warning'] = process_warning
                                    elif key == 'proc.info':
                                        process_info.append(val)
                                        updates['process_info'] = process_info
                                    elif key == 'proc.rc':
                                        process_rc = int(val)
                                        updates['process_rc'] = process_rc
                                        if process_rc != 0:
                                            updates['status'] = DATA_META['STATUS_ERROR']
                                    elif key == 'proc.progress':
                                        process_progress = int(float(val) * 100)
                                        updates['process_progress'] = process_progress
                                else:
                                    output[key] = val
                                    updates['output'] = output

                        if updates:
                            await self.update_data_status(**updates)
                            # Process meta fields are collected in listener, so we can clear them.
                            process_error, process_warning, process_info = [], [], []

                        if process_rc > 0:
                            log_file.close()
                            json_file.close()
                            await self._send_manager_command(ExecutorProtocol.FINISH, extra_fields={
                                ExecutorProtocol.FINISH_PROCESS_RC: process_rc
                            })
                            return

                        # Debug output
                        # Not referenced in Data object
                        json_file.write(line)
                        json_file.flush()

                except ValueError as ex:
                    # Ignore if not JSON
                    log_file.write(line)
                    log_file.flush()

        except MemoryError as ex:
            logger.error("Out of memory:\n\n{}".format(ex))

        except IOError as ex:
            # TODO: if ex.errno == 28: no more free space
            raise ex
        finally:
            # Store results
            log_file.close()
            json_file.close()

        return_code = await self.end()

        if process_rc < return_code:
            process_rc = return_code

        # send a notification to the executor listener that we're done
        finish_fields = {
            ExecutorProtocol.FINISH_PROCESS_RC: process_rc
        }
        if spawn_processes and process_rc == 0:
            finish_fields[ExecutorProtocol.FINISH_SPAWN_PROCESSES] = spawn_processes
            finish_fields[ExecutorProtocol.FINISH_EXPORTED_FILES] = self.exported_files_mapper

        return finish_fields

Response: Execute the script and save the results.


Instruction:
Implement a function in Python 3 to
return the total number of bytes of requested unprocessed replicas.

Response:
def get_total_size_of_queued_replicas():
    """Return the total number of bytes of requested, unprocessed replicas."""
    return (
        d1_gmn.app.models.ReplicationQueue.objects.filter(
            local_replica__info__status__status='queued'
        ).aggregate(Sum('size'))['size__sum']
        or 0
    )


Instruction:
Here you have a function in Python 3, explain what it does
def add_to_replication_queue(source_node_urn, sysmeta_pyxb):
    replica_info_model = d1_gmn.app.models.replica_info(
        status_str='queued', source_node_urn=source_node_urn
    )
    local_replica_model = d1_gmn.app.models.local_replica(
        pid=d1_common.xml.get_req_val(sysmeta_pyxb.identifier),
        replica_info_model=replica_info_model,
    )
    d1_gmn.app.models.replication_queue(
        local_replica_model=local_replica_model, size=sysmeta_pyxb.size
    )

Response: Add a replication request issued by a CN to a queue that is processed asynchronously.


Instruction:
Can you create a Python 3 function that
adds standard arguments for DataONE utilities to a command line parser.

Response:
def add_arguments(parser, doc_str, add_base_url=True):
    """Add standard arguments for DataONE utilities to a command line parser."""
    parser.description = doc_str
    parser.formatter_class = argparse.RawDescriptionHelpFormatter
    parser.add_argument("--debug", action="store_true", help="Debug level logging")
    parser.add_argument(
        "--cert-pub",
        dest="cert_pem_path",
        action="store",
        default=django.conf.settings.CLIENT_CERT_PATH,
        help="Path to PEM formatted public key of certificate",
    )
    parser.add_argument(
        "--cert-key",
        dest="cert_key_path",
        action="store",
        default=django.conf.settings.CLIENT_CERT_PRIVATE_KEY_PATH,
        help="Path to PEM formatted private key of certificate",
    )
    parser.add_argument(
        "--public", action="store_true", help="Do not use certificate even if available"
    )
    parser.add_argument(
        "--disable-server-cert-validation",
        action="store_true",
        help="Do not validate the TLS/SSL server side certificate of the source node (insecure)",
    )
    parser.add_argument(
        "--timeout",
        type=float,
        action="store",
        default=DEFAULT_TIMEOUT_SEC,
        help="Timeout for DataONE API calls to the source MN",
    )
    parser.add_argument(
        "--retries",
        type=int,
        action="store",
        default=DEFAULT_RETRY_COUNT,
        help="Retry DataONE API calls that raise HTTP level exceptions",
    )
    parser.add_argument(
        "--page-size",
        type=int,
        action="store",
        default=DEFAULT_PAGE_SIZE,
        help="Number of objects to retrieve in each list method API call to source MN",
    )
    parser.add_argument(
        "--major",
        type=int,
        action="store",
        help="Skip automatic detection of API major version and use the provided version",
    )
    parser.add_argument(
        "--max-concurrent",
        type=int,
        action="store",
        default=DEFAULT_MAX_CONCURRENT_TASK_COUNT,
        help="Max number of concurrent DataONE API",
    )
    if not add_base_url:
        parser.add_argument(
            "--baseurl",
            action="store",
            default=django.conf.settings.DATAONE_ROOT,
            help="Remote MN or CN BaseURL",
        )
    else:
        parser.add_argument("baseurl", help="Remote MN or CN BaseURL")


Instruction:
Implement a Python 3 function for
getting a list of all members of a Resource Map.

Response:
def get_resource_map_members(pid):
    """``pid`` is the PID of a Resource Map or the PID of a member of a Resource Map."""
    if d1_gmn.app.did.is_resource_map_db(pid):
        return get_resource_map_members_by_map(pid)
    elif d1_gmn.app.did.is_resource_map_member(pid):
        return get_resource_map_members_by_member(pid)
    else:
        raise d1_common.types.exceptions.InvalidRequest(
            0, 'Not a Resource Map or Resource Map member. pid="{}"'.format(pid)
        )


Instruction:
Make a summary of the following Python 3 code
def set_with_conversion(self, variable, value_string):
        self._assert_valid_variable(variable)
        try:
            v = ast.literal_eval(value_string)
        except (ValueError, SyntaxError):
            v = value_string
        if v is None or v == "none":
            self._variables[variable] = None
        else:
            try:
                type_converter = variable_type_map[variable]
                value_string = self._validate_variable_type(
                    value_string, type_converter
                )
                value = type_converter(value_string)
                self._variables[variable] = value
            except ValueError:
                raise d1_cli.impl.exceptions.InvalidArguments(
                    "Invalid value for {}: {}".format(variable, value_string)
                )

Response: Convert user supplied string to Python type.


Instruction:
Explain what the following Python 3 code does
def log_setup(is_debug=False, is_multiprocess=False):
    format_str = (
        '%(asctime)s %(name)s %(module)s:%(lineno)d %(process)4d %(levelname)-8s %(message)s'
        if is_multiprocess
        else '%(asctime)s %(name)s %(module)s:%(lineno)d %(levelname)-8s %(message)s'
    )
    formatter = logging.Formatter(format_str, '%Y-%m-%d %H:%M:%S')
    console_logger = logging.StreamHandler(sys.stdout)
    console_logger.setFormatter(formatter)
    logging.getLogger('').addHandler(console_logger)
    if is_debug:
        logging.getLogger('').setLevel(logging.DEBUG)
    else:
        logging.getLogger('').setLevel(logging.INFO)

Response: Setup a standardized log format for the DataONE Python stack.


Instruction:
Can you tell what is the following Python 3 function doing
def get_content_type(content_type):
    m = email.message.Message()
    m['Content-Type'] = content_type
    return m.get_content_type()

Response: Extract the MIME type value from a content type string.


Instruction:
Create a Python 3 function for
merging two nested dicts.

Response:
def nested_update(d, u):
    """Merge two nested dicts.

    Nested dicts are sometimes used for representing various recursive structures. When
    updating such a structure, it may be convenient to present the updated data as a
    corresponding recursive structure. This function will then apply the update.

    Args:
      d: dict
        dict that will be updated in-place. May or may not contain nested dicts.

      u: dict
        dict with contents that will be merged into ``d``. May or may not contain
        nested dicts.

    """
    for k, v in list(u.items()):
        if isinstance(v, collections.Mapping):
            r = nested_update(d.get(k, {}), v)
            d[k] = r
        else:
            d[k] = u[k]
    return d


Instruction:
How would you explain what the following Python 3 function does
def print_logging():
    root_logger = logging.getLogger()
    old_level_list = [h.level for h in root_logger.handlers]
    for h in root_logger.handlers:
        h.setLevel(logging.WARN)
    log_format = logging.Formatter('%(message)s')
    stream_handler = logging.StreamHandler(sys.stdout)
    stream_handler.setFormatter(log_format)
    stream_handler.setLevel(logging.DEBUG)
    root_logger.addHandler(stream_handler)
    yield
    root_logger.removeHandler(stream_handler)
    for h, level in zip(root_logger.handlers, old_level_list):
        h.setLevel(level)

Response:
Context manager that temporarily suppress additional information such as timestamps
    when writing to loggers.


Instruction:
Here you have a function in Python 3, explain what it does
def save_json(py_obj, json_path):
    with open(json_path, 'w', encoding='utf-8') as f:
        f.write(serialize_to_normalized_pretty_json(py_obj))

Response:
Serialize a native object to JSON and save it normalized pretty printed to a
    file.


Instruction:
Create a Python 3 function to
serialize a native object to normalized pretty printed JSON.

Response:
def serialize_to_normalized_pretty_json(py_obj):
    """Serialize a native object to normalized, pretty printed JSON.

    The JSON string is normalized by sorting any dictionary keys.

    Args:
      py_obj: object
        Any object that can be represented in JSON. Some types, such as datetimes are
        automatically converted to strings.

    Returns:
      str: normalized, pretty printed JSON string.

    """
    return json.dumps(py_obj, sort_keys=True, indent=2, cls=ToJsonCompatibleTypes)


Instruction:
Make a summary of the following Python 3 code
def serialize_to_normalized_compact_json(py_obj):
    return json.dumps(
        py_obj, sort_keys=True, separators=(',', ':'), cls=ToJsonCompatibleTypes
    )

Response: Serialize a native object to normalized compact JSON.


Instruction:
Can you generate the documentation for the following Python 3 function
def format_sec_to_dhm(sec):
    rem_int, s_int = divmod(int(sec), 60)
    rem_int, m_int, = divmod(rem_int, 60)
    d_int, h_int, = divmod(rem_int, 24)
    return '{}d{:02d}h{:02d}m'.format(d_int, h_int, m_int)

Response: Format seconds to days hours minutes.


Instruction:
Here you have a function in Python 3, explain what it does
def count(self, event_str, inc_int=1):
        self._event_dict.setdefault(event_str, 0)
        self._event_dict[event_str] += inc_int

Response: Counts the number of times an event occurs in the event dict.


Instruction:
Can you create a Python 3 function that
counts an event and write a message to a logger.

Response:
def log_and_count(self, event_str, msg_str=None, inc_int=None):
        """Count an event and write a message to a logger.

        Args:
          event_str: str
            The name of an event to count. Used as a key in the event dict. The same
            name will be used in the summary. This also becomes a part of the message
            logged by this function.

          msg_str: str
            Optional message with details about the events. The message is only written
            to the log. While the ``event_str`` functions as a key and must remain the
            same for the same type of event, ``log_str`` may change between calls.

          inc_int: int
            Optional argument to increase the count for the event by more than 1.

        """
        logger.info(
            ' - '.join(map(str, [v for v in (event_str, msg_str, inc_int) if v]))
        )
        self.count(event_str, inc_int or 1)


Instruction:
How would you explain what the following Python 3 function does
def dump_to_log(self):
        if self._event_dict:
            logger.info('Events:')
            for event_str, count_int in sorted(self._event_dict.items()):
                logger.info('  {}: {}'.format(event_str, count_int))
        else:
            logger.info('No Events')

Response: Write summary to logger.


Instruction:
Can you implement a function in Python 3 that
clears the database. Used for testing and debugging.

Response:
def delete_all_from_db():
    """Clear the database.

    Used for testing and debugging.

    """
    # The models.CASCADE property is set on all ForeignKey fields, so tables can
    # be deleted in any order without breaking constraints.
    for model in django.apps.apps.get_models():
        model.objects.all().delete()


Instruction:
Write a Python 3 function for
getting a query parameter uniformly for GET and POST requests.

Response:
def get_query_param(request, key):
    """Get query parameter uniformly for GET and POST requests."""
    value = request.query_params.get(key) or request.data.get(key)
    if value is None:
        raise KeyError()
    return value


Instruction:
Given the following Python 3 function, write the documentation
def futurize_module(module_path, show_diff, write_update):
    logging.info('Futurizing module... path="{}"'.format(module_path))
    ast_tree = back_to_the_futurize(module_path)
    return d1_dev.util.update_module_file_ast(
        ast_tree, module_path, show_diff, write_update
    )

Response: 2to3 uses AST, not Baron.


Instruction:
Implement a Python 3 function for
removing comments from a single line import statement.

Response:
def _remove_single_line_import_comments(r):
    """We previously used more groups for the import statements and named each group."""
    logging.info('Removing single line import comments')
    import_r, remaining_r = split_by_last_import(r)
    new_import_r = redbaron.NodeList()
    for i, v in enumerate(import_r):
        if 1 < i < len(import_r) - 2:
            if not (
                import_r[i - 2].type != 'comment'
                and v.type == 'comment'
                and import_r[i + 2].type != 'comment'
            ) or _is_keep_comment(v):
                new_import_r.append(v)
        else:
            new_import_r.append(v)
    return new_import_r + remaining_r


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _update_init_all(module_path, r):
    module_dir_path = os.path.split(module_path)[0]
    module_list = []
    for item_name in os.listdir(module_dir_path):
        item_path = os.path.join(module_dir_path, item_name)
        if os.path.isfile(item_path) and item_name in ('__init__.py', 'setup.py'):
            continue
        if os.path.isfile(item_path) and not item_name.endswith('.py'):
            continue
        # if os.path.isdir(item_path) and not os.path.isfile(
        #     os.path.join(item_path, '__init__.py')
        # ):
        #   continue
        if os.path.isdir(item_path):
            continue
        module_list.append(re.sub(r'.py$', '', item_name).encode('utf-8'))
    module_literal_str = str(sorted(module_list))

    assignment_node_list = r('AssignmentNode', recursive=False)
    for n in assignment_node_list:
        if n.type == 'assignment' and n.target.value == '__all__':
            n.value = module_literal_str
            break
    else:
        r.node_list.append(
            redbaron.RedBaron('__all__ = {}\n'.format(module_literal_str))
        )
    return r

Response: Add or update __all__ in __init__. py file.


Instruction:
Can you create a Python 3 function that
removes any __all__ in __init__. py file.

Response:
def _remove_init_all(r):
    """Remove any __all__ in __init__.py file."""
    new_r = redbaron.NodeList()
    for n in r.node_list:
        if n.type == 'assignment' and n.target.value == '__all__':
            pass
        else:
            new_r.append(n)
    return new_r


Instruction:
Can you generate a brief explanation for the following Python 3 code
def get_object_list_json(request):
    # TODO: Add to documentation
    if "f" in request.GET:
        field_list = request.GET.getlist("f")
    else:
        field_list = None

    result_dict = d1_gmn.app.views.util.query_object_list(request, "object_list_json")
    result_dict["fields"] = field_list
    result_dict["objects"] = d1_gmn.app.sysmeta_extract.extract_values_query(
        result_dict["query"], field_list
    )
    del result_dict["query"]

    return django.http.HttpResponse(
        d1_common.util.serialize_to_normalized_pretty_json(result_dict),
        d1_common.const.CONTENT_TYPE_JSON,
    )

Response: Return a JSON representation of the object list.


Instruction:
Write a Python 3 function that can
configure logging to send log records to the master.

Response:
def configure_logging(emit_list):
    """Configure logging to send log records to the master."""
    if 'sphinx' in sys.modules:
        module_base = 'resolwe.flow.executors'
    else:
        module_base = 'executors'
    logging_config = dict(
        version=1,
        formatters={
            'json_formatter': {
                '()': JSONFormatter
            },
        },
        handlers={
            'redis': {
                'class': module_base + '.logger.RedisHandler',
                'formatter': 'json_formatter',
                'level': logging.INFO,
                'emit_list': emit_list
            },
            'console': {
                'class': 'logging.StreamHandler',
                'level': logging.WARNING
            },

        },
        root={
            'handlers': ['redis', 'console'],
            'level': logging.DEBUG,
        },
        loggers={
            # Don't use redis logger to prevent circular dependency.
            module_base + '.manager_comm': {
                'level': 'INFO',
                'handlers': ['console'],
                'propagate': False,
            },
        },
    )

    dictConfig(logging_config)


Instruction:
Here you have a function in Python 3, explain what it does
def format(self, record):
        data = record.__dict__.copy()

        data['data_id'] = DATA['id']
        data['data_location_id'] = DATA_LOCATION['id']
        data['hostname'] = socket.gethostname()

        # Get relative path, so listener can reconstruct the path to the actual code.
        data['pathname'] = os.path.relpath(data['pathname'], os.path.dirname(__file__))

        # Exception and Traceback cannot be serialized.
        data['exc_info'] = None

        # Ensure logging message is instantiated to a string.
        data['msg'] = str(data['msg'])

        return json.dumps(data)

Response: Dump the record to JSON.


Instruction:
How would you code a function in Python 3 to
send log message to the listener.

Response:
def emit(self, record):
        """Send log message to the listener."""
        future = asyncio.ensure_future(send_manager_command(
            ExecutorProtocol.LOG,
            extra_fields={
                ExecutorProtocol.LOG_MESSAGE: self.format(record),
            },
            expect_reply=False
        ))
        self.emit_list.append(future)


Instruction:
How would you explain what the following Python 3 function does
async def is_object_synced_to_cn(self, client, pid):
        status = await client.describe(pid)
        if status == 200:
            self.progress_logger.event("SciObj already synced on CN")
            return True
        elif status == 404:
            self.progress_logger.event("SciObj has not synced to CN")
            return False
        self.progress_logger.event(
            "CNRead.describe() returned unexpected status code. "
            'pid="{}" status="{}"'.format(pid, status)
        )
        return True

Response: Check if object with pid has successfully synced to CN.


Instruction:
Can you implement a function in Python 3 that
issues a notification and request for sync for object with pid to the CN.

Response:
async def send_synchronization_request(self, client, pid):
        """Issue a notification and request for sync for object with {pid} to the CN."""
        # Skip CN call for debugging
        # status = 200
        status = await client.synchronize(pid)
        if status == 200:
            self.progress_logger.event("Issued sync request, CN accepted")
        else:
            self.progress_logger.event(
                "CNRead.synchronize() returned unexpected status code. "
                'pid="{}" status="{}"'.format(pid, status)
            )


Instruction:
Implement a function in Python 3 to
create a new secret returning its handle.

Response:
def create_secret(self, value, contributor, metadata=None, expires=None):
        """Create a new secret, returning its handle.

        :param value: Secret value to store
        :param contributor: User owning the secret
        :param metadata: Optional metadata dictionary (must be JSON serializable)
        :param expires: Optional date/time of expiry (defaults to None, which means that
            the secret never expires)
        :return: Secret handle
        """
        if metadata is None:
            metadata = {}

        secret = self.create(
            value=value,
            contributor=contributor,
            metadata=metadata,
            expires=expires,
        )
        return str(secret.handle)


Instruction:
Write a Python 3 function that can
retrieve an existing secret s value.

Response:
def get_secret(self, handle, contributor):
        """Retrieve an existing secret's value.

        :param handle: Secret handle
        :param contributor: User instance to perform contributor validation,
            which means that only secrets for the given contributor will be
            looked up.
        """
        queryset = self.all()
        if contributor is not None:
            queryset = queryset.filter(contributor=contributor)
        secret = queryset.get(handle=handle)
        return secret.value


Instruction:
Write a Python 3 function for
returning json schema for json validation.

Response:
def validation_schema(name):
    """Return json schema for json validation."""
    schemas = {
        'processor': 'processSchema.json',
        'descriptor': 'descriptorSchema.json',
        'field': 'fieldSchema.json',
        'type': 'typeSchema.json',
    }

    if name not in schemas:
        raise ValueError()

    field_schema_file = finders.find('flow/{}'.format(schemas['field']), all=True)[0]
    with open(field_schema_file, 'r') as fn:
        field_schema = fn.read()

    if name == 'field':
        return json.loads(field_schema.replace('{{PARENT}}', ''))

    schema_file = finders.find('flow/{}'.format(schemas[name]), all=True)[0]
    with open(schema_file, 'r') as fn:
        schema = fn.read()

    return json.loads(schema.replace('{{FIELD}}', field_schema).replace('{{PARENT}}', '/field'))


Instruction:
Can you generate the documentation for the following Python 3 function
def validate_schema(instance, schema, test_required=True, data_location=None,
                    skip_missing_data=False):
    """Check if DictField values are consistent with our data types.

    Perform basic JSON schema validation and our custom validations:

      * check that required fields are given (if `test_required` is set
        to ``True``)
      * check if ``basic:file:`` and ``list:basic:file`` fields match
        regex given in schema (only if ``validate_regex`` is defined in
        schema for coresponding fields) and exists (only if
        ``data_location`` is given)
      * check if directories referenced in ``basic:dir:`` and
        ``list:basic:dir``fields exist (only if ``data_location`` is
        given)
      * check that referenced ``Data`` objects (in ``data:<data_type>``
        and  ``list:data:<data_type>`` fields) exists and are of type
        ``<data_type>``
      * check that referenced ``Storage`` objects (in ``basic:json``
        fields) exists

    :param list instance: Instance to be validated
    :param list schema: Schema for validation
    :param bool test_required: Flag for testing if all required fields
        are present. It is usefule if validation is run before ``Data``
        object is finished and there are some field stil missing
        (default: ``False``)
    :param :class:`~resolwe.flow.models.data.DataLocation` data_location:
        data location used for checking if files and directories exist
        (default: ``None``)
    :param bool skip_missing_data: Don't raise an error if referenced
        ``Data`` object does not exist
    :rtype: None
    :raises ValidationError: if ``instance`` doesn't match schema
        defined in ``schema``

    """
    from .storage import Storage  # Prevent circular import.

    path_prefix = None
    if data_location:
        path_prefix = data_location.get_path()

    def validate_refs(field):
        """Validate reference paths."""
        for ref_filename in field.get('refs', []):
            ref_path = os.path.join(path_prefix, ref_filename)
            if not os.path.exists(ref_path):
                raise ValidationError("Path referenced in `refs` ({}) does not exist.".format(ref_path))
            if not (os.path.isfile(ref_path) or os.path.isdir(ref_path)):
                raise ValidationError(
                    "Path referenced in `refs` ({}) is neither a file or directory.".format(ref_path))

    def validate_file(field, regex):
        """Validate file name (and check that it exists)."""
        filename = field['file']

        if regex and not re.search(regex, filename):
            raise ValidationError(
                "File name {} does not match regex {}".format(filename, regex))

        if path_prefix:
            path = os.path.join(path_prefix, filename)
            if not os.path.exists(path):
                raise ValidationError("Referenced path ({}) does not exist.".format(path))
            if not os.path.isfile(path):
                raise ValidationError("Referenced path ({}) is not a file.".format(path))

            validate_refs(field)

    def validate_dir(field):
        """Check that dirs and referenced files exists."""
        dirname = field['dir']

        if path_prefix:
            path = os.path.join(path_prefix, dirname)
            if not os.path.exists(path):
                raise ValidationError("Referenced path ({}) does not exist.".format(path))
            if not os.path.isdir(path):
                raise ValidationError("Referenced path ({}) is not a directory.".format(path))

            validate_refs(field)

    def validate_data(data_pk, type_):
        """Check that `Data` objects exist and is of right type."""
        from .data import Data  # prevent circular import

        data_qs = Data.objects.filter(pk=data_pk).values('process__type')
        if not data_qs.exists():
            if skip_missing_data:
                return

            raise ValidationError(
                "Referenced `Data` object does not exist (id:{})".format(data_pk))
        data = data_qs.first()
        if not data['process__type'].startswith(type_):
            raise ValidationError(
                "Data object of type `{}` is required, but type `{}` is given. "
                "(id:{})".format(type_, data['process__type'], data_pk))

    def validate_range(value, interval, name):
        """Check that given value is inside the specified range."""
        if not interval:
            return

        if value < interval[0] or value > interval[1]:
            raise ValidationError(
                "Value of field '{}' is out of range. It should be between {} and {}.".format(
                    name, interval[0], interval[1]
                )
            )

    is_dirty = False
    dirty_fields = []
    for _schema, _fields, _ in iterate_schema(instance, schema):
        name = _schema['name']
        is_required = _schema.get('required', True)

        if test_required and is_required and name not in _fields:
            is_dirty = True
            dirty_fields.append(name)

        if name in _fields:
            field = _fields[name]
            type_ = _schema.get('type', "")

            # Treat None as if the field is missing.
            if not is_required and field is None:
                continue

            try:
                jsonschema.validate([{"type": type_, "value": field}], TYPE_SCHEMA)
            except jsonschema.exceptions.ValidationError as ex:
                raise ValidationError(ex.message)

            choices = [choice['value'] for choice in _schema.get('choices', [])]
            allow_custom_choice = _schema.get('allow_custom_choice', False)
            if choices and not allow_custom_choice and field not in choices:
                raise ValidationError(
                    "Value of field '{}' must match one of predefined choices. "
                    "Current value: {}".format(name, field)
                )

            if type_ == 'basic:file:':
                validate_file(field, _schema.get('validate_regex'))

            elif type_ == 'list:basic:file:':
                for obj in field:
                    validate_file(obj, _schema.get('validate_regex'))

            elif type_ == 'basic:dir:':
                validate_dir(field)

            elif type_ == 'list:basic:dir:':
                for obj in field:
                    validate_dir(obj)

            elif type_ == 'basic:json:' and not Storage.objects.filter(pk=field).exists():
                raise ValidationError(
                    "Referenced `Storage` object does not exist (id:{})".format(field))

            elif type_.startswith('data:'):
                validate_data(field, type_)

            elif type_.startswith('list:data:'):
                for data_id in field:
                    validate_data(data_id, type_[5:])  # remove `list:` from type

            elif type_ == 'basic:integer:' or type_ == 'basic:decimal:':
                validate_range(field, _schema.get('range'), name)

            elif type_ == 'list:basic:integer:' or type_ == 'list:basic:decimal:':
                for obj in field:
                    validate_range(obj, _schema.get('range'), name)

    try:
        # Check that schema definitions exist for all fields
        for _, _ in iterate_fields(instance, schema):
            pass
    except KeyError as ex:
        raise ValidationError(str(ex))

    if is_dirty:
        dirty_fields = ['"{}"'.format(field) for field in dirty_fields]
        raise DirtyError("Required fields {} not given.".format(', '.join(dirty_fields)))

Response: Validate that the given instance and schema are consistent with our data types.


Instruction:
Can you tell what is the following Python 3 function doing
def _hydrate_values(output, output_schema, data):
    def hydrate_path(file_name):
        """Hydrate file paths."""
        from resolwe.flow.managers import manager

        class HydratedPath(str):
            """String wrapper, which also stores the original filename."""

            __slots__ = ('data_id', 'file_name')

            def __new__(cls, value=''):
                """Initialize hydrated path."""
                hydrated = str.__new__(cls, value)
                hydrated.data_id = data.id
                hydrated.file_name = file_name
                return hydrated

        return HydratedPath(manager.get_executor().resolve_data_path(data, file_name))

    def hydrate_storage(storage_id):
        """Hydrate storage fields."""
        from .storage import LazyStorageJSON  # Prevent circular import.

        return LazyStorageJSON(pk=storage_id)

    for field_schema, fields in iterate_fields(output, output_schema):
        name = field_schema['name']
        value = fields[name]
        if 'type' in field_schema:
            if field_schema['type'].startswith('basic:file:'):
                value['file'] = hydrate_path(value['file'])
                value['refs'] = [hydrate_path(ref) for ref in value.get('refs', [])]

            elif field_schema['type'].startswith('list:basic:file:'):
                for obj in value:
                    obj['file'] = hydrate_path(obj['file'])
                    obj['refs'] = [hydrate_path(ref) for ref in obj.get('refs', [])]

            if field_schema['type'].startswith('basic:dir:'):
                value['dir'] = hydrate_path(value['dir'])
                value['refs'] = [hydrate_path(ref) for ref in value.get('refs', [])]

            elif field_schema['type'].startswith('list:basic:dir:'):
                for obj in value:
                    obj['dir'] = hydrate_path(obj['dir'])
                    obj['refs'] = [hydrate_path(ref) for ref in obj.get('refs', [])]

            elif field_schema['type'].startswith('basic:json:'):
                fields[name] = hydrate_storage(value)

            elif field_schema['type'].startswith('list:basic:json:'):
                fields[name] = [hydrate_storage(storage_id) for storage_id in value]

Response: Hydrate basic file and basic. json values.


Instruction:
Can you generate the documentation for the following Python 3 function
def hydrate_input_references(input_, input_schema, hydrate_values=True):
    from .data import Data  # prevent circular import

    for field_schema, fields in iterate_fields(input_, input_schema):
        name = field_schema['name']
        value = fields[name]
        if 'type' in field_schema:
            if field_schema['type'].startswith('data:'):
                if value is None:
                    continue

                try:
                    data = Data.objects.get(id=value)
                except Data.DoesNotExist:
                    fields[name] = {}
                    continue

                output = copy.deepcopy(data.output)
                if hydrate_values:
                    _hydrate_values(output, data.process.output_schema, data)
                output["__id"] = data.id
                output["__type"] = data.process.type
                output["__descriptor"] = data.descriptor
                output["__entity_name"] = None
                output["__output_schema"] = data.process.output_schema

                entity = data.entity_set.values('name').first()
                if entity:
                    output["__entity_name"] = entity['name']

                fields[name] = output

            elif field_schema['type'].startswith('list:data:'):
                outputs = []
                for val in value:
                    if val is None:
                        continue

                    try:
                        data = Data.objects.get(id=val)
                    except Data.DoesNotExist:
                        outputs.append({})
                        continue

                    output = copy.deepcopy(data.output)
                    if hydrate_values:
                        _hydrate_values(output, data.process.output_schema, data)

                    output["__id"] = data.id
                    output["__type"] = data.process.type
                    output["__descriptor"] = data.descriptor
                    output["__output_schema"] = data.process.output_schema

                    entity = data.entity_set.values('name').first()
                    if entity:
                        output["__entity_name"] = entity['name']

                    outputs.append(output)

                fields[name] = outputs

Response: Hydrate input_ with linked data.


Instruction:
How would you explain what the following Python 3 function does
def hydrate_input_uploads(input_, input_schema, hydrate_values=True):
    from resolwe.flow.managers import manager

    files = []
    for field_schema, fields in iterate_fields(input_, input_schema):
        name = field_schema['name']
        value = fields[name]
        if 'type' in field_schema:
            if field_schema['type'] == 'basic:file:':
                files.append(value)

            elif field_schema['type'] == 'list:basic:file:':
                files.extend(value)

    urlregex = re.compile(r'^(https?|ftp)://[-A-Za-z0-9\+&@#/%?=~_|!:,.;]*[-A-Za-z0-9\+&@#/%=~_|]')
    for value in files:
        if 'file_temp' in value:
            if isinstance(value['file_temp'], str):
                # If file_temp not url, hydrate path.
                if not urlregex.search(value['file_temp']):
                    value['file_temp'] = manager.get_executor().resolve_upload_path(value['file_temp'])
            else:
                # Something very strange happened.
                value['file_temp'] = 'Invalid value for file_temp in DB'

Response: Hydrate input basic : upload types with upload location.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def hydrate_size(data, force=False):
    from .data import Data  # prevent circular import

    def get_dir_size(path):
        """Get directory size."""
        total_size = 0
        for dirpath, _, filenames in os.walk(path):
            for file_name in filenames:
                file_path = os.path.join(dirpath, file_name)
                if not os.path.isfile(file_path):  # Skip all "not normal" files (links, ...)
                    continue
                total_size += os.path.getsize(file_path)
        return total_size

    def get_refs_size(obj, obj_path):
        """Calculate size of all references of ``obj``.

        :param dict obj: Data object's output field (of type file/dir).
        :param str obj_path: Path to ``obj``.
        """
        total_size = 0
        for ref in obj.get('refs', []):
            ref_path = data.location.get_path(filename=ref)
            if ref_path in obj_path:
                # It is a common case that ``obj['file']`` is also contained in
                # one of obj['ref']. In that case, we need to make sure that it's
                # size is not counted twice:
                continue
            if os.path.isfile(ref_path):
                total_size += os.path.getsize(ref_path)
            elif os.path.isdir(ref_path):
                total_size += get_dir_size(ref_path)

        return total_size

    def add_file_size(obj):
        """Add file size to the basic:file field."""
        if data.status in [Data.STATUS_DONE, Data.STATUS_ERROR] and 'size' in obj and not force:
            return

        path = data.location.get_path(filename=obj['file'])
        if not os.path.isfile(path):
            raise ValidationError("Referenced file does not exist ({})".format(path))

        obj['size'] = os.path.getsize(path)
        obj['total_size'] = obj['size'] + get_refs_size(obj, path)

    def add_dir_size(obj):
        """Add directory size to the basic:dir field."""
        if data.status in [Data.STATUS_DONE, Data.STATUS_ERROR] and 'size' in obj and not force:
            return

        path = data.location.get_path(filename=obj['dir'])
        if not os.path.isdir(path):
            raise ValidationError("Referenced dir does not exist ({})".format(path))

        obj['size'] = get_dir_size(path)
        obj['total_size'] = obj['size'] + get_refs_size(obj, path)

    data_size = 0
    for field_schema, fields in iterate_fields(data.output, data.process.output_schema):
        name = field_schema['name']
        value = fields[name]
        if 'type' in field_schema:
            if field_schema['type'].startswith('basic:file:'):
                add_file_size(value)
                data_size += value.get('total_size', 0)
            elif field_schema['type'].startswith('list:basic:file:'):
                for obj in value:
                    add_file_size(obj)
                    data_size += obj.get('total_size', 0)
            elif field_schema['type'].startswith('basic:dir:'):
                add_dir_size(value)
                data_size += value.get('total_size', 0)
            elif field_schema['type'].startswith('list:basic:dir:'):
                for obj in value:
                    add_dir_size(obj)
                    data_size += obj.get('total_size', 0)

    data.size = data_size

Response: Populates the size of all files and directories of all objects in the object.


Instruction:
How would you explain what the following Python 3 function does
def render_descriptor(data):
    if not data.descriptor_schema:
        return

    # Set default values
    for field_schema, field, path in iterate_schema(data.descriptor, data.descriptor_schema.schema, 'descriptor'):
        if 'default' in field_schema and field_schema['name'] not in field:
            dict_dot(data, path, field_schema['default'])

Response:
Render data descriptor.

    The rendering is based on descriptor schema and input context.

    :param data: data instance
    :type data: :class:`resolwe.flow.models.Data` or :class:`dict`


Instruction:
Make a summary of the following Python 3 code
def render_template(process, template_string, context):
    from resolwe.flow.managers import manager

    # Get the appropriate expression engine. If none is defined, do not evaluate
    # any expressions.
    expression_engine = process.requirements.get('expression-engine', None)
    if not expression_engine:
        return template_string

    return manager.get_expression_engine(expression_engine).evaluate_block(template_string, context)

Response: Render a template using the specified expression engine.


Instruction:
Can you tell what is the following Python 3 function doing
def json_path_components(path):
    if isinstance(path, str):
        path = path.split('.')

    return list(path)

Response: Convert a JSON path to individual path components.


Instruction:
Explain what the following Python 3 code does
def validate_process_subtype(supertype_name, supertype, subtype_name, subtype):
    errors = []
    for item in supertype:
        # Ensure that the item exists in subtype and has the same schema.
        for subitem in subtype:
            if item['name'] != subitem['name']:
                continue

            for key in set(item.keys()) | set(subitem.keys()):
                if key in ('label', 'description'):
                    # Label and description can differ.
                    continue
                elif key == 'required':
                    # A non-required item can be made required in subtype, but not the
                    # other way around.
                    item_required = item.get('required', True)
                    subitem_required = subitem.get('required', False)

                    if item_required and not subitem_required:
                        errors.append("Field '{}' is marked as required in '{}' and optional in '{}'.".format(
                            item['name'],
                            supertype_name,
                            subtype_name,
                        ))
                elif item.get(key, None) != subitem.get(key, None):
                    errors.append("Schema for field '{}' in type '{}' does not match supertype '{}'.".format(
                        item['name'],
                        subtype_name,
                        supertype_name
                    ))

            break
        else:
            errors.append("Schema for type '{}' is missing supertype '{}' field '{}'.".format(
                subtype_name,
                supertype_name,
                item['name']
            ))

    return errors

Response: Perform process subtype validation.


Instruction:
Create a Python 3 function to
perform process type validation.

Response:
def validate_process_types(queryset=None):
    """Perform process type validation.

    :param queryset: Optional process queryset to validate
    :return: A list of validation error strings
    """
    if not queryset:
        from .process import Process
        queryset = Process.objects.all()

    processes = {}
    for process in queryset:
        dict_dot(
            processes,
            process.type.replace(':', '.') + '__schema__',
            process.output_schema
        )

    errors = []
    for path, key, value in iterate_dict(processes, exclude=lambda key, value: key == '__schema__'):
        if '__schema__' not in value:
            continue

        # Validate with any parent types.
        for length in range(len(path), 0, -1):
            parent_type = '.'.join(path[:length] + ['__schema__'])
            try:
                parent_schema = dict_dot(processes, parent_type)
            except KeyError:
                continue

            errors += validate_process_subtype(
                supertype_name=':'.join(path[:length]),
                supertype=parent_schema,
                subtype_name=':'.join(path + [key]),
                subtype=value['__schema__']
            )

    return errors


Instruction:
Write a Python 3 function for
filling empty optional fields in input with default values.

Response:
def fill_with_defaults(process_input, input_schema):
    """Fill empty optional fields in input with default values."""
    for field_schema, fields, path in iterate_schema(process_input, input_schema):
        if 'default' in field_schema and field_schema['name'] not in fields:
            dict_dot(process_input, path, field_schema['default'])


Instruction:
Implement a function in Python 3 to
format the internal value.

Response:
def to_internal_value(self, data):
        """Format the internal value."""
        # When setting the contributor, it may be passed as an integer.
        if isinstance(data, dict) and isinstance(data.get('id', None), int):
            data = data['id']
        elif isinstance(data, int):
            pass
        else:
            raise ValidationError("Contributor must be an integer or a dictionary with key 'id'")

        return self.Meta.model.objects.get(pk=data)


Instruction:
Can you generate the documentation for the following Python 3 function
def extract_descriptor(self, obj):
        descriptor = []

        def flatten(current):
            """Flatten descriptor."""
            if isinstance(current, dict):
                for key in current:
                    flatten(current[key])
            elif isinstance(current, list):
                for val in current:
                    flatten(val)
            elif isinstance(current, (int, bool, float, str)):
                descriptor.append(str(current))

        flatten(obj.descriptor)

        return descriptor

Response: Extract data from the descriptor.


Instruction:
Make a summary of the following Python 3 code
def _serialize_items(self, serializer, kind, items):
        if self.request and self.request.query_params.get('hydrate_{}'.format(kind), False):
            serializer = serializer(items, many=True, read_only=True)
            serializer.bind(kind, self)
            return serializer.data
        else:
            return [item.id for item in items]

Response: Return serialized items or list of ids depending on hydrate_XXX query param.


Instruction:
Can you write a function in Python 3 where it
returns serialized list of entity names on data that user has view permission on.

Response:
def get_entity_names(self, data):
        """Return serialized list of entity names on data that user has `view` permission on."""
        entities = self._filter_queryset('view_entity', data.entity_set.all())
        return list(entities.values_list('name', flat=True))


Instruction:
How would you implement a function in Python 3 that
returns serialized list of collection objects on data that user has view permission on.

Response:
def get_collections(self, data):
        """Return serialized list of collection objects on data that user has `view` permission on."""
        collections = self._filter_queryset('view_collection', data.collection_set.all())

        from .collection import CollectionSerializer

        class CollectionWithoutDataSerializer(WithoutDataSerializerMixin, CollectionSerializer):
            """Collection without data field serializer."""

        return self._serialize_items(CollectionWithoutDataSerializer, 'collections', collections)


Instruction:
Write a Python 3 function that can
return serialized list of entity objects on data that user has view permission on.

Response:
def get_entities(self, data):
        """Return serialized list of entity objects on data that user has `view` permission on."""
        entities = self._filter_queryset('view_entity', data.entity_set.all())

        from .entity import EntitySerializer

        class EntityWithoutDataSerializer(WithoutDataSerializerMixin, EntitySerializer):
            """Entity without data field serializer."""

        return self._serialize_items(EntityWithoutDataSerializer, 'entities', entities)