Instruction:
How would you implement a function in Python 3 that
creates an input node for the given frequency and adds it to the pipeline.

Response:
def _make_inputnode(self, frequency):
        """
        Generates an input node for the given frequency. It also adds implicit
        file format conversion nodes to the pipeline.

        Parameters
        ----------
        frequency : str
            The frequency (i.e. 'per_session', 'per_visit', 'per_subject' or
            'per_study') of the input node to retrieve
        """
        # Check to see whether there are any outputs for the given frequency
        inputs = list(self.frequency_inputs(frequency))
        # Get list of input names for the requested frequency, addding fields
        # to hold iterator IDs
        input_names = [i.name for i in inputs]
        input_names.extend(self.study.FREQUENCIES[frequency])
        if not input_names:
            raise ArcanaError(
                "No inputs to '{}' pipeline for requested freqency '{}'"
                .format(self.name, frequency))
        # Generate input node and connect it to appropriate nodes
        inputnode = self.add('{}_inputnode'.format(frequency),
                             IdentityInterface(fields=input_names))
        # Loop through list of nodes connected to study data specs and
        # connect them to the newly created input node
        for input in inputs:  # @ReservedAssignment
            # Keep track of previous conversion nodes to avoid replicating the
            # conversion for inputs that are used in multiple places
            prev_conv_nodes = {}
            for (node, node_in, format,  # @ReservedAssignment @IgnorePep8
                 conv_kwargs) in self._input_conns[input.name]:
                # If fileset formats differ between study and pipeline
                # inputs create converter node (if one hasn't been already)
                # and connect input to that before connecting to inputnode
                if self.requires_conversion(input, format):
                    try:
                        conv = format.converter_from(input.format,
                                                     **conv_kwargs)
                    except ArcanaNoConverterError as e:
                        e.msg += (
                            "which is required to convert '{}' from {} to {} "
                            "for '{}' input of '{}' node".format(
                                input.name, input.format, format, node_in,
                                node.name))
                        raise e
                    try:
                        in_node = prev_conv_nodes[format.name]
                    except KeyError:
                        in_node = prev_conv_nodes[format.name] = self.add(
                            'conv_{}_to_{}_format'.format(input.name,
                                                          format.name),
                            conv.interface,
                            inputs={conv.input: (inputnode, input.name)},
                            requirements=conv.requirements,
                            mem_gb=conv.mem_gb,
                            wall_time=conv.wall_time)
                    in_node_out = conv.output
                else:
                    in_node = inputnode
                    in_node_out = input.name
                self.connect(in_node, in_node_out, node, node_in)
        # Connect iterator inputs
        for iterator, conns in self._iterator_conns.items():
            # Check to see if this is the right frequency for the iterator
            # input, i.e. if it is the only iterator for this frequency
            if self.study.FREQUENCIES[frequency] == (iterator,):
                for (node, node_in, format) in conns:  # @ReservedAssignment
                    self.connect(inputnode, iterator, node, node_in)
        return inputnode


Instruction:
Can you generate the documentation for the following Python 3 function
def _make_outputnode(self, frequency):
        # Check to see whether there are any outputs for the given frequency
        outputs = list(self.frequency_outputs(frequency))
        if not outputs:
            raise ArcanaError(
                "No outputs to '{}' pipeline for requested freqency '{}'"
                .format(self.name, frequency))
        # Get list of output names for the requested frequency, addding fields
        # to hold iterator IDs
        output_names = [o.name for o in outputs]
        # Generate output node and connect it to appropriate nodes
        outputnode = self.add('{}_outputnode'.format(frequency),
                              IdentityInterface(fields=output_names))
        # Loop through list of nodes connected to study data specs and
        # connect them to the newly created output node
        for output in outputs:  # @ReservedAssignment
            (node, node_out, format,  # @ReservedAssignment @IgnorePep8
             conv_kwargs) = self._output_conns[output.name]
            # If fileset formats differ between study and pipeline
            # outputs create converter node (if one hasn't been already)
            # and connect output to that before connecting to outputnode
            if self.requires_conversion(output, format):
                conv = output.format.converter_from(format, **conv_kwargs)
                node = self.add(
                    'conv_{}_from_{}_format'.format(output.name, format.name),
                    conv.interface,
                    inputs={conv.input: (node, node_out)},
                    requirements=conv.requirements,
                    mem_gb=conv.mem_gb,
                    wall_time=conv.wall_time)
                node_out = conv.output
            self.connect(node, node_out, outputnode, output.name)
        return outputnode

Response: Creates an output node for the given frequency.


Instruction:
Write a Python 3 function that can
generate provenance information for the pipeline and returns a dictionary containing the provenance information for the pipeline and the pipeline itself.

Response:
def _gen_prov(self):
        """
        Extracts provenance information from the pipeline into a PipelineProv
        object

        Returns
        -------
        prov : dict[str, *]
            A dictionary containing the provenance information to record
            for the pipeline
        """
        # Export worfklow graph to node-link data format
        wf_dict = nx_json.node_link_data(self.workflow._graph)
        # Replace references to Node objects with the node's provenance
        # information and convert to a dict organised by node name to allow it
        # to be compared more easily. Also change link node-references from
        # node index to node ID so it is not dependent on the order the nodes
        # are written to the dictionary (which for Python < 3.7 is guaranteed
        # to be the same between identical runs)
        for link in wf_dict['links']:
            if int(networkx_version.split('.')[0]) < 2:  # @UndefinedVariable
                link['source'] = wf_dict['nodes'][link['source']]['id'].name
                link['target'] = wf_dict['nodes'][link['target']]['id'].name
            else:
                link['source'] = link['source'].name
                link['target'] = link['target'].name
        wf_dict['nodes'] = {n['id'].name: n['id'].prov
                            for n in wf_dict['nodes']}
        # Roundtrip to JSON to convert any tuples into lists so dictionaries
        # can be compared directly
        wf_dict = json.loads(json.dumps(wf_dict))
        dependency_versions = {d: extract_package_version(d)
                               for d in ARCANA_DEPENDENCIES}
        pkg_versions = {'arcana': __version__}
        pkg_versions.update((k, v) for k, v in dependency_versions.items()
                            if v is not None)
        prov = {
            '__prov_version__': PROVENANCE_VERSION,
            'name': self.name,
            'workflow': wf_dict,
            'study': self.study.prov,
            'pkg_versions': pkg_versions,
            'python_version': sys.version,
            'joined_ids': self._joined_ids()}
        return prov


Instruction:
Write a Python 3 function that can
construct the provenance record that would be saved in the given node.

Response:
def expected_record(self, node):
        """
        Constructs the provenance record that would be saved in the given node
        if the pipeline was run on the current state of the repository

        Parameters
        ----------
        node : arcana.repository.tree.TreeNode
            A node of the Tree representation of the study data stored in the
            repository (i.e. a Session, Visit, Subject or Tree node)

        Returns
        -------
        expected_record : arcana.provenance.Record
            The record that would be produced if the pipeline is run over the
            study tree.
        """
        exp_inputs = {}
        # Get checksums/values of all inputs that would have been used in
        # previous runs of an equivalent pipeline to compare with that saved
        # in provenance to see if any have been updated.
        for inpt in self.inputs:  # @ReservedAssignment
            # Get iterators present in the input that aren't in this node
            # and need to be joined
            iterators_to_join = (self.iterators(inpt.frequency) -
                                 self.iterators(node.frequency))
            if not iterators_to_join:
                # No iterators to join so we can just extract the checksums
                # of the corresponding input
                exp_inputs[inpt.name] = inpt.collection.item(
                    node.subject_id, node.visit_id).checksums
            elif len(iterators_to_join) == 1:
                # Get list of checksums dicts for each node of the input
                # frequency that relates to the current node
                exp_inputs[inpt.name] = [
                    inpt.collection.item(n.subject_id, n.visit_id).checksums
                    for n in node.nodes(inpt.frequency)]
            else:
                # In the case where the node is the whole treee and the input
                # is per_seession, we need to create a list of lists to match
                # how the checksums are joined in the processor
                exp_inputs[inpt.name] = []
                for subj in node.subjects:
                    exp_inputs[inpt.name].append([
                        inpt.collection.item(s.subject_id,
                                             s.visit_id).checksums
                        for s in subj.sessions])
        # Get checksums/value for all outputs of the pipeline. We are assuming
        # that they exist here (otherwise they will be None)
        exp_outputs = {
            o.name: o.collection.item(node.subject_id, node.visit_id).checksums
            for o in self.outputs}
        exp_prov = copy(self.prov)
        if PY2:
            # Need to convert to unicode strings for Python 2
            exp_inputs = json.loads(json.dumps(exp_inputs))
            exp_outputs = json.loads(json.dumps(exp_outputs))
        exp_prov['inputs'] = exp_inputs
        exp_prov['outputs'] = exp_outputs
        exp_prov['joined_ids'] = self._joined_ids()
        return Record(
            self.name, node.frequency, node.subject_id, node.visit_id,
            self.study.name, exp_prov)


Instruction:
Write a Python 3 script to
add the subject and visit ids used to generate the derivatives iff there are any joins over them in the pipeline

Response:
def _joined_ids(self):
        """
        Adds the subjects/visits used to generate the derivatives iff there are
        any joins over them in the pipeline
        """
        joined_prov = {}
        if self.joins_subjects:
            joined_prov['subject_ids'] = list(self.study.subject_ids)
        if self.joins_visits:
            joined_prov['visit_ids'] = list(self.study.visit_ids)
        return joined_prov


Instruction:
Make a summary of the following Python 3 code
def tree(self, subject_ids=None, visit_ids=None, **kwargs):
        # Find all data present in the repository (filtered by the passed IDs)
        return Tree.construct(
            self, *self.find_data(subject_ids=subject_ids,
                                  visit_ids=visit_ids), **kwargs)

Response: Return the hierarchical tree of subject and session and fileset related to the given subject and visit IDs.


Instruction:
Write a Python 3 function for
returning the tree for the given subject and visit ID filters and the IDs that were actually returned by the tree.

Response:
def cached_tree(self, subject_ids=None, visit_ids=None, fill=False):
        """
        Access the repository tree and caches it for subsequent
        accesses

        Parameters
        ----------
        subject_ids : list(str)
            List of subject IDs with which to filter the tree with. If
            None all are returned
        visit_ids : list(str)
            List of visit IDs with which to filter the tree with. If
            None all are returned
        fill : bool
            Create empty sessions for any that are missing in the
            subject_id x visit_id block. Typically only used if all
            the inputs to the study are coming from different repositories
            to the one that the derived products are stored in

        Returns
        -------
        tree : arcana.repository.Tree
            A hierarchical tree of subject, vist, session information and that
            of the filesets and fields they contain
        """
        if subject_ids is not None:
            subject_ids = frozenset(subject_ids)
        if visit_ids is not None:
            visit_ids = frozenset(visit_ids)
        try:
            tree = self._cache[subject_ids][visit_ids]
        except KeyError:
            if fill:
                fill_subjects = subject_ids
                fill_visits = visit_ids
            else:
                fill_subjects = fill_visits = None
            tree = self.tree(
                subject_ids=subject_ids, visit_ids=visit_ids,
                fill_visits=fill_visits, fill_subjects=fill_subjects)
            # Save the tree within the cache under the given subject/
            # visit ID filters and the IDs that were actually returned
            self._cache[subject_ids][visit_ids] = self._cache[
                frozenset(tree.subject_ids)][frozenset(tree.visit_ids)] = tree
        return tree


Instruction:
Can you generate the documentation for the following Python 3 function
def resource_names(self, repo_type):
        try:
            names = self._resource_names[repo_type]
        except KeyError:
            names = [self.name, self.name.upper()]
        return names

Response: Returns a list of resource names for this format on a given repository type.


Instruction:
Can you tell what is the following Python 3 function doing
def default_aux_file_paths(self, primary_path):
        return dict((n, primary_path[:-len(self.ext)] + ext)
                    for n, ext in self.aux_files.items())

Response: Get the default paths for auxiliary files relative to the path of the primary file


Instruction:
How would you explain what the following Python 3 function does
def assort_files(self, candidates):
        by_ext = defaultdict(list)
        for path in candidates:
            by_ext[split_extension(path)[1].lower()].append(path)
        try:
            primary_file = by_ext[self.ext]
        except KeyError:
            raise ArcanaFileFormatError(
                "No files match primary file extension of {} out of "
                "potential candidates of {}"
                .format(self, "', '".join(candidates)))
        if not primary_file:
            raise ArcanaFileFormatError(
                "No potential files for primary file of {}".format(self))
        elif len(primary_file) > 1:
            raise ArcanaFileFormatError(
                "Multiple potential files for '{}' primary file of {}"
                .format("', '".join(primary_file), self))
        else:
            primary_file = primary_file[0]
        aux_files = {}
        for aux_name, aux_ext in self.aux_files.items():
            try:
                aux = by_ext[aux_ext]
            except KeyError:
                raise ArcanaFileFormatError(
                    "No files match auxiliary file extension '{}' of {} out of"
                    " potential candidates of {}"
                    .format(aux_ext, self, "', '".join(candidates)))
            if len(aux) > 1:
                raise ArcanaFileFormatError(
                    "Multiple potential files for '{}' auxiliary file ext. "
                    "({}) of {}".format("', '".join(aux),
                                        self))
            aux_files[aux_name] = aux[0]
        return primary_file, aux_files

Response: Assorts candidate files into primary and auxiliary files.


Instruction:
Given the following Python 3 function, write the documentation
def matches(self, fileset):
        if fileset._resource_name is not None:
            return (fileset._resource_name in self.resource_names(
                fileset.repository.type))
        elif self.directory:
            if op.isdir(fileset.path):
                if self.within_dir_exts is None:
                    return True
                else:
                    # Get set of all extensions in the directory
                    return self.within_dir_exts == frozenset(
                        split_extension(f)[1] for f in os.listdir(fileset.path)
                        if not f.startswith('.'))
            else:
                return False
        else:
            if op.isfile(fileset.path):
                all_paths = [fileset.path] + fileset._potential_aux_files
                try:
                    primary_path = self.assort_files(all_paths)[0]
                except ArcanaFileFormatError:
                    return False
                else:
                    return primary_path == fileset.path
            else:
                return False

Response: Checks to see if the format matches the given fileset.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def set_converter(self, file_format, converter):
        self._converters[file_format.name] = (file_format, converter)

Response: Register a Converter and the FileFormat that it can convert from from


Instruction:
Can you generate the documentation for the following Python 3 function
def parse_url(url):
    scheme, dest = url.split('://')
    host = None
    ssl_context = None
    if scheme == 'elk':
        host, port = dest.split(':') if ':' in dest else (dest, 2101)
    elif scheme == 'elks':
        host, port = dest.split(':') if ':' in dest else (dest, 2601)
        ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
        ssl_context.verify_mode = ssl.CERT_NONE
    elif scheme == 'serial':
        host, port = dest.split(':') if ':' in dest else (dest, 115200)
    else:
        raise ValueError("Invalid scheme '%s'" % scheme)
    return (scheme, host, int(port), ssl_context)

Response: Parse an Elk connection string into a tuple.


Instruction:
Can you tell what is the following Python 3 function doing
def pretty_const(value):
    words = value.split('_')
    pretty = words[0].capitalize()
    for word in words[1:]:
        pretty += ' ' + word.lower()
    return pretty

Response: Make a constant pretty for printing in GUI


Instruction:
How would you code a function in Python 3 to
return name of user.

Response:
def username(elk, user_number):
    """Return name of user."""
    if user_number >= 0 and user_number < elk.users.max_elements:
        return elk.users[user_number].name
    if user_number == 201:
        return "*Program*"
    if user_number == 202:
        return "*Elk RP*"
    if user_number == 203:
        return "*Quick arm*"
    return ""


Instruction:
Write a Python 3 script to
connect to the ElkM1.

Response:
async def _connect(self, connection_lost_callbk=None):
        """Asyncio connection to Elk."""
        self.connection_lost_callbk = connection_lost_callbk
        url = self._config['url']
        LOG.info("Connecting to ElkM1 at %s", url)
        scheme, dest, param, ssl_context = parse_url(url)
        conn = partial(Connection, self.loop, self._connected,
                       self._disconnected, self._got_data, self._timeout)
        try:
            if scheme == 'serial':
                await serial_asyncio.create_serial_connection(
                    self.loop, conn, dest, baudrate=param)
            else:
                await asyncio.wait_for(self.loop.create_connection(
                    conn, host=dest, port=param, ssl=ssl_context), timeout=30)
        except (ValueError, OSError, asyncio.TimeoutError) as err:
            LOG.warning("Could not connect to ElkM1 (%s). Retrying in %d seconds",
                        err, self._connection_retry_timer)
            self.loop.call_later(self._connection_retry_timer, self.connect)
            self._connection_retry_timer = 2 * self._connection_retry_timer \
                if self._connection_retry_timer < 32 else 60


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _connected(self, transport, conn):
        LOG.info("Connected to ElkM1")
        self._conn = conn
        self._transport = transport
        self._connection_retry_timer = 1
        if url_scheme_is_secure(self._config['url']):
            self._conn.write_data(self._config['userid'], raw=True)
            self._conn.write_data(self._config['password'], raw=True)
        self.call_sync_handlers()
        if not self._config['url'].startswith('serial://'):
            self._heartbeat = self.loop.call_later(120, self._reset_connection)

Response: Login and sync the ElkM1 panel to memory.


Instruction:
Implement a function in Python 3 to
send a message to the Elk panel.

Response:
def send(self, msg):
        """Send a message to Elk panel."""
        if self._conn:
            self._conn.write_data(msg.message, msg.response_command)


Instruction:
Make a summary of the following Python 3 code
def sync(self):
        self.elk.send(az_encode())
        self.elk.send(zd_encode())
        self.elk.send(zp_encode())
        self.elk.send(zs_encode())
        self.get_descriptions(TextDescriptions.ZONE.value)

Response: Retrieve zones from ElkM1


Instruction:
Write a Python 3 function for
raising an exception if the model uses a history manager and historical model history_id is not a UUIDField.

Response:
def has_offline_historical_manager_or_raise(self):
        """Raises an exception if model uses a history manager and
        historical model history_id is not a UUIDField.

        Note: expected to use edc_model.HistoricalRecords instead of
        simple_history.HistoricalRecords.
        """
        try:
            model = self.instance.__class__.history.model
        except AttributeError:
            model = self.instance.__class__
        field = [field for field in model._meta.fields if field.name == "history_id"]
        if field and not isinstance(field[0], UUIDField):
            raise OfflineHistoricalManagerError(
                f"Field 'history_id' of historical model "
                f"'{model._meta.app_label}.{model._meta.model_name}' "
                "must be an UUIDfield. "
                "For history = HistoricalRecords() use edc_model.HistoricalRecords instead of "
                "simple_history.HistoricalRecords(). "
                f"See '{self.instance._meta.app_label}.{self.instance._meta.model_name}'."
            )


Instruction:
How would you explain what the following Python 3 function does
def primary_key_field(self):
        return [field for field in self.instance._meta.fields if field.primary_key][0]

Response: Return the primary key field.


Instruction:
Write a Python 3 script to
serialize the object to an OutgoingTransaction object and saves it to the OutgoingTransaction model.

Response:
def to_outgoing_transaction(self, using, created=None, deleted=None):
        """ Serialize the model instance to an AES encrypted json object
        and saves the json object to the OutgoingTransaction model.
        """
        OutgoingTransaction = django_apps.get_model(
            "django_collect_offline", "OutgoingTransaction"
        )
        created = True if created is None else created
        action = INSERT if created else UPDATE
        timestamp_datetime = (
            self.instance.created if created else self.instance.modified
        )
        if not timestamp_datetime:
            timestamp_datetime = get_utcnow()
        if deleted:
            timestamp_datetime = get_utcnow()
            action = DELETE
        outgoing_transaction = None
        if self.is_serialized:
            hostname = socket.gethostname()
            outgoing_transaction = OutgoingTransaction.objects.using(using).create(
                tx_name=self.instance._meta.label_lower,
                tx_pk=getattr(self.instance, self.primary_key_field.name),
                tx=self.encrypted_json(),
                timestamp=timestamp_datetime.strftime("%Y%m%d%H%M%S%f"),
                producer=f"{hostname}-{using}",
                action=action,
                using=using,
            )
        return outgoing_transaction


Instruction:
Here you have a function in Python 3, explain what it does
def encrypted_json(self):
        json = serialize(objects=[self.instance])
        encrypted_json = Cryptor().aes_encrypt(json, LOCAL_MODE)
        return encrypted_json

Response: Returns an encrypted json serialized from self.


Instruction:
Can you tell what is the following Python 3 function doing
def map_req(self, requirement):
        if isinstance(self._packages_map, dict):
            local_name = self._packages_map.get(requirement, requirement.name)
        else:
            local_name = self._packages_map(requirement)
        return local_name

Response:
Maps the name of an Requirement class to the name of the corresponding
            module in the environment.


Instruction:
How would you explain what the following Python 3 function does
def map_version(self, requirement, local_version):
        if isinstance(self._versions_map, dict):
            version = self._versions_map.get(requirement, {}).get(
                local_version, local_version)
        else:
            version = self._versions_map(requirement, local_version)
        return version

Response: Maps a local version name to one recognised by the Requirement class


Instruction:
Can you generate a brief explanation for the following Python 3 code
def init_tag_processors(self):
        '''Register the TagProcessors that are bundled with doxytag2zealdb.'''
        register = self.register_tag_processor

        register('class', classTagProcessor(**self.opts))
        register('file', fileTagProcessor(**self.opts))
        register('namespace', namespaceTagProcessor(**self.opts))
        register('struct', structTagProcessor(**self.opts))
        register('union', unionTagProcessor(**self.opts))
        register('function', functionTagProcessor(**self.opts))
        register('define', defineTagProcessor(**self.opts))
        register('enumeration', enumerationTagProcessor(**self.opts))
        register('enumvalue', enumvalueTagProcessor(**self.opts))
        register('typedef', typedefTagProcessor(**self.opts))
        register('variable', variableTagProcessor(**self.opts))

Response: Register the TagProcessors that are bundled with doxytag2zealdb.


Instruction:
Can you tell what is the following Python 3 function doing
def process(self):
        '''Run all tag processors.'''
        for tag_proc in self.tag_procs:
            before_count = self.entry_count
            self.run_tag_processor(tag_proc)
            after_count = self.entry_count

            if self.verbose:
                print('Inserted %d entries for "%s" tag processor' % (
                    after_count - before_count, tag_proc), file=sys.stderr)

        if self.verbose:
            print('Inserted %d entries overall' % self.entry_count,
                  file=sys.stderr)

Response: Run all tag processors.


Instruction:
Make a summary of the following Python 3 code
def run_tag_processor(self, tag_proc_name):
        '''Run a tag processor.

        Args:
            tag_proc_name: A string key that maps to the TagProcessor to run.
        '''
        tag_processor = self.tag_procs[tag_proc_name]

        for tag in tag_processor.find(self.soup):
            self.process_tag(tag_proc_name, tag)

Response: Run a tag processor.


Instruction:
Can you generate the documentation for the following Python 3 function
def process_tag(self, tag_proc_name, tag):
        '''Process a tag with a tag processor and insert a DB entry.

        Args:
            tag_proc_name: A string key that maps to the TagProcessor to use.
            tag: A BeautifulSoup Tag to process.
        '''
        tag_processor = self.tag_procs[tag_proc_name]

        db_entry = (tag_processor.get_name(tag),
                    tag_processor.get_entry_type(tag),
                    tag_processor.get_filename(tag))

        self.zeal_db.insert(*db_entry)

        self.entry_count += 1

Response: Process a tag with a tag processor and insert a DB entry.


Instruction:
Implement a function in Python 3 to
wrap to itertools. groupby that returns a list of each group and returns a list of each group

Response:
def groupby(iterable, key=0, filter=None):
    """
    wrapper to itertools.groupby that returns a list of each group, rather
    than a generator and accepts integers or strings as the key and
    automatically converts them to callables with itemgetter(key)

    Arguments:
        iterable: iterable
        key: string, int or callable that tells how to group

    Returns:
        an iterable where each item is the key and a *list* of that
        group. (itertools.groupby returns a generator of that group).

    e.g. groupby(iterable, 0)
    """
    if isinstance(key, (basestring, int)):
        key = itemgetter(key)
    elif isinstance(key, (tuple, list)):
        key = itemgetter(*key)
    for label, grp in igroupby(iterable, key):
        yield label, list(grp)


Instruction:
Here you have a function in Python 3, explain what it does
def groups_of(n, iterable):
    args = [iter(iterable)] * n
    for x in izip_longest(*args):
        yield [v for v in x if v is not None]

Response: Generator that yields n elements from iterable.


Instruction:
Explain what the following Python 3 code does
def run(self, *pipelines, **kwargs):
        if not pipelines:
            raise ArcanaUsageError("No pipelines provided to {}.run"
                                   .format(self))
        # Get filter kwargs  (NB: in Python 3 they could be in the kwarg list)
        subject_ids = kwargs.pop('subject_ids', None)
        visit_ids = kwargs.pop('visit_ids', None)
        session_ids = kwargs.pop('session_ids', None)
        clean_work_dir = kwargs.pop('clean_work_dir',
                                    self._clean_work_dir_between_runs)
        required_outputs = kwargs.pop('required_outputs', repeat(None))
        # Create name by combining pipelines
        name = '_'.join(p.name for p in pipelines)
        # Clean work dir if required
        if clean_work_dir:
            workflow_work_dir = op.join(self.work_dir, name)
            if op.exists(workflow_work_dir):
                shutil.rmtree(workflow_work_dir)
        # Trim the end of very large names to avoid problems with
        # workflow names exceeding system limits.
        name = name[:WORKFLOW_MAX_NAME_LEN]
        workflow = pe.Workflow(name=name, base_dir=self.work_dir)

        # Generate filter array to optionally restrict the run to certain
        # subject and visit IDs.
        tree = self.study.tree
        # Create maps from the subject|visit IDs to an index used to represent
        # them in the filter array
        subject_inds = {s.id: i for i, s in enumerate(tree.subjects)}
        visit_inds = {v.id: i for i, v in enumerate(tree.visits)}
        if not subject_ids and not visit_ids and not session_ids:
            # No filters applied so create a full filter array
            filter_array = np.ones((len(subject_inds), len(visit_inds)),
                                   dtype=bool)
        else:
            # Filters applied so create an empty filter array and populate
            # from filter lists
            filter_array = np.zeros((len(subject_inds), len(visit_inds)),
                                    dtype=bool)
            if subject_ids is not None:
                for subj_id in subject_ids:
                    filter_array[subject_inds[subj_id], :] = True
            if visit_ids is not None:
                for visit_id in visit_ids:
                    filter_array[:, visit_inds[visit_id]] = True
            if session_ids is not None:
                for subj_id, visit_id in session_ids:
                    filter_array[subject_inds[subj_id],
                                 visit_inds[visit_id]] = True
            if not filter_array.any():
                raise ArcanaUsageError(
                    "Provided filters:\n" +
                    ("  subject_ids: {}\n".format(', '.join(subject_ids))
                     if subject_ids is not None else '') +
                    ("  visit_ids: {}\n".format(', '.join(visit_ids))
                     if visit_ids is not None else '') +
                    ("  session_ids: {}\n".format(', '.join(session_ids))
                     if session_ids is not None else '') +
                    "Did not match any sessions in the project:\n" +
                    "  subject_ids: {}\n".format(', '.join(subject_inds)) +
                    "  visit_ids: {}\n".format(', '.join(visit_inds)))

        # Stack of pipelines to process in reverse order of required execution
        stack = OrderedDict()

        def push_on_stack(pipeline, filt_array, req_outputs):
            if req_outputs is None:
                req_outputs = pipeline.output_names
            if pipeline.name in stack:
                # Pop pipeline from stack in order to add it to the end of the
                # stack and ensure it is run before all downstream pipelines
                prev_pipeline, prev_req_outputs, prev_filt_array = stack.pop(
                    pipeline.name)
                if pipeline is not prev_pipeline and pipeline != prev_pipeline:
                    raise ArcanaDesignError(
                        "Attempting to run two different pipelines with the "
                        "same name, {} and {}".format(pipeline, prev_pipeline))
                # Combined required outputs and filter array with previous
                # references to the pipeline
                req_outputs = copy(req_outputs)
                req_outputs.update(prev_req_outputs)
                filt_array = filt_array | prev_filt_array
            # If the pipeline to process contains summary outputs (i.e. 'per-
            # subject|visit|study' frequency), then we need to "dialate" the
            # filter array to include IDs across the scope of the study, e.g.
            # all subjects for per-vist, or all visits for per-subject.
            output_freqs = set(pipeline.output_frequencies)
            dialated_filt_array = self._dialate_array(filt_array,
                                                      pipeline.joins)
            added = dialated_filt_array ^ filt_array
            if added.any():
                filt_array = dialated_filt_array
                # Invert the index dictionaries to get index-to-ID maps
                inv_subject_inds = {v: k for k, v in subject_inds.items()}
                inv_visit_inds = {v: k for k, v in visit_inds.items()}
                logger.warning(
                    "Dialated filter array used to process '{}' pipeline to "
                    "include {} subject/visit IDs due to its '{}' summary "
                    "outputs ".format(
                        pipeline.name,
                        ', '.join('({},{})'.format(inv_subject_inds[s],
                                                   inv_visit_inds[v])
                                  for s, v in zip(*np.nonzero(added))),
                        "' and '".join(output_freqs)))

            stack[pipeline.name] = pipeline, req_outputs, filt_array
            # Recursively add all prerequisites to stack
            for prq_getter, prq_req_outputs in pipeline.prerequisites.items():
                try:
                    prereq = pipeline.study.pipeline(
                        prq_getter, prq_req_outputs)
                    push_on_stack(prereq, filt_array, prq_req_outputs)
                except (ArcanaMissingDataException,
                        ArcanaOutputNotProducedException) as e:
                    e.msg += ("{}, which are required as inputs to the '{}' "
                              "pipeline to produce '{}'".format(
                                  pipeline.name, "', '".join(req_outputs)))
                    raise e

        # Add all primary pipelines to the stack along with their prereqs
        for pipeline, req_outputs in zip(pipelines, required_outputs):
            push_on_stack(pipeline, filter_array, req_outputs)

        # Iterate through stack of required pipelines from upstream to
        # downstream
        for pipeline, req_outputs, flt_array in reversed(list(stack.values())):
            try:
                self._connect_pipeline(
                    pipeline, req_outputs, workflow, subject_inds, visit_inds,
                    flt_array, **kwargs)
            except ArcanaNoRunRequiredException:
                logger.info("Not running '{}' pipeline as its outputs "
                            "are already present in the repository"
                            .format(pipeline.name))
        # Save complete graph for debugging purposes
#         workflow.write_graph(graph2use='flat', format='svg')
#         print('Graph saved in {} directory'.format(os.getcwd()))
        # Actually run the generated workflow
        result = workflow.run(plugin=self._plugin)
        # Reset the cached tree of filesets in the repository as it will
        # change after the pipeline has run.
        self.study.clear_caches()
        return result

Response: Runs the given pipelines and returns the final report node which can be used to generate the next set of nodes.


Instruction:
Can you implement a function in Python 3 that
connects a pipeline to an overarching workflow that sets up iterators over subjects and visits.

Response:
def _connect_pipeline(self, pipeline, required_outputs, workflow,
                          subject_inds, visit_inds, filter_array, force=False):
        """
        Connects a pipeline to a overarching workflow that sets up iterators
        over subjects|visits present in the repository (if required) and
        repository source and sink nodes

        Parameters
        ----------
        pipeline : Pipeline
            The pipeline to connect
        required_outputs : set[str] | None
            The outputs required to be produced by this pipeline. If None all
            are deemed to be required
        workflow : nipype.pipeline.engine.Workflow
            The overarching workflow to connect the pipeline to
        subject_inds : dct[str, int]
            A mapping of subject ID to row index in the filter array
        visit_inds : dct[str, int]
            A mapping of visit ID to column index in the filter array
        filter_array : 2-D numpy.array[bool]
            A two-dimensional boolean array, where rows correspond to
            subjects and columns correspond to visits in the repository. True
            values represent a combination of subject & visit ID to include
            in the current round of processing. Note that if the 'force'
            flag is not set, sessions won't be reprocessed unless the
            save provenance doesn't match that of the given pipeline.
        force : bool | 'all'
            A flag to force the processing of all sessions in the filter
            array, regardless of whether the parameters|pipeline used
            to generate existing data matches the given pipeline
        """
        if self.reprocess == 'force':
            force = True
        # Close-off construction of the pipeline and created, input and output
        # nodes and provenance dictionary
        pipeline.cap()
        # Prepend prerequisite pipelines to complete workflow if they need
        # to be (re)processed
        final_nodes = []
        # The array that represents the subject/visit pairs for which any
        # prerequisite pipeline will be (re)processed, and which therefore
        # needs to be included in the processing of the current pipeline. Row
        # indices correspond to subjects and column indices visits
        prqs_to_process_array = np.zeros((len(subject_inds), len(visit_inds)),
                                         dtype=bool)
        # The array that represents the subject/visit pairs for which any
        # prerequisite pipeline will be skipped due to missing inputs. Row
        # indices correspond to subjects and column indices visits
        prqs_to_skip_array = np.zeros((len(subject_inds), len(visit_inds)),
                                      dtype=bool)
        for getter_name in pipeline.prerequisites:
            prereq = pipeline.study.pipeline(getter_name)
            if prereq.to_process_array.any():
                final_nodes.append(prereq.node('final'))
                prqs_to_process_array |= prereq.to_process_array
            prqs_to_skip_array |= prereq.to_skip_array
        # Get list of sessions that need to be processed (i.e. if
        # they don't contain the outputs of this pipeline)
        to_process_array, to_protect_array, to_skip_array = self._to_process(
            pipeline, required_outputs, prqs_to_process_array,
            prqs_to_skip_array, filter_array, subject_inds, visit_inds, force)
        # Store the arrays signifying which nodes to process, protect or skip
        # so they can be passed to downstream pipelines
        pipeline.to_process_array = to_process_array
        pipeline.to_protect_array = to_protect_array
        pipeline.to_skip_array = to_skip_array
        # Check to see if there are any sessions to process
        if not to_process_array.any():
            raise ArcanaNoRunRequiredException(
                "No sessions to process for '{}' pipeline"
                .format(pipeline.name))
        # Set up workflow to run the pipeline, loading and saving from the
        # repository
        workflow.add_nodes([pipeline._workflow])
        # If prerequisite pipelines need to be processed, connect their
        # "final" nodes to the initial node of this pipeline to ensure that
        # they are all processed before this pipeline is run.
        if final_nodes:
            prereqs = pipeline.add('prereqs', Merge(len(final_nodes)))
            for i, final_node in enumerate(final_nodes, start=1):
                workflow.connect(final_node, 'out', prereqs, 'in{}'.format(i))
        else:
            prereqs = None
        # Construct iterator structure over subjects and sessions to be
        # processed
        iter_nodes = self._iterate(pipeline, to_process_array, subject_inds,
                                   visit_inds)
        sources = {}
        # Loop through each frequency present in the pipeline inputs and
        # create a corresponding source node
        for freq in pipeline.input_frequencies:
            try:
                inputs = list(pipeline.frequency_inputs(freq))
            except ArcanaMissingDataException as e:
                raise ArcanaMissingDataException(
                    str(e) + ", which is required for pipeline '{}'".format(
                        pipeline.name))
            inputnode = pipeline.inputnode(freq)
            sources[freq] = source = pipeline.add(
                '{}_source'.format(freq),
                RepositorySource(
                    i.collection for i in inputs),
                inputs=({'prereqs': (prereqs, 'out')} if prereqs is not None
                         else {}))
            # Connect iter_nodes to source and input nodes
            for iterator in pipeline.iterators(freq):
                pipeline.connect(iter_nodes[iterator], iterator, source,
                                 iterator)
                pipeline.connect(source, iterator, inputnode,
                                 iterator)
            for input in inputs:  # @ReservedAssignment
                pipeline.connect(source, input.suffixed_name,
                                 inputnode, input.name)
        deiter_nodes = {}

        def deiter_node_sort_key(it):
            """
            If there are two iter_nodes (i.e. both subject and visit ID) and
            one depends on the other (i.e. if the visit IDs per subject
            vary and vice-versa) we need to ensure that the dependent
            iterator is deiterated (joined) first.
            """
            return iter_nodes[it].itersource is None

        # Connect all outputs to the repository sink, creating a new sink for
        # each frequency level (i.e 'per_session', 'per_subject', 'per_visit',
        # or 'per_study')
        for freq in pipeline.output_frequencies:
            outputs = list(pipeline.frequency_outputs(freq))
            if pipeline.iterators(freq) - pipeline.iterators():
                raise ArcanaDesignError(
                    "Doesn't make sense to output '{}', which are of '{}' "
                    "frequency, when the pipeline only iterates over '{}'"
                    .format("', '".join(o.name for o in outputs), freq,
                            "', '".join(pipeline.iterators())))
            outputnode = pipeline.outputnode(freq)
            # Connect filesets/fields to sink to sink node, skipping outputs
            # that are study inputs
            to_connect = {o.suffixed_name: (outputnode, o.name)
                          for o in outputs if o.is_spec}
            # Connect iterators to sink node
            to_connect.update(
                {i: (iter_nodes[i], i) for i in pipeline.iterators()})
            # Connect checksums/values from sources to sink node in order to
            # save in provenance, joining where necessary
            for input_freq in pipeline.input_frequencies:
                checksums_to_connect = [
                    i.checksum_suffixed_name
                    for i in pipeline.frequency_inputs(input_freq)]
                if not checksums_to_connect:
                    # Rare case of a pipeline with no inputs only iter_nodes
                    # that will only occur in unittests in all likelihood
                    continue
                # Loop over iterators that need to be joined, i.e. that are
                # present in the input frequency but not the output frequency
                # and create join nodes
                source = sources[input_freq]
                for iterator in (pipeline.iterators(input_freq) -
                                  pipeline.iterators(freq)):
                    join = pipeline.add(
                        '{}_to_{}_{}_checksum_join'.format(
                            input_freq, freq, iterator),
                        IdentityInterface(
                            checksums_to_connect),
                        inputs={
                            tc: (source, tc) for tc in checksums_to_connect},
                        joinsource=iterator,
                        joinfield=checksums_to_connect)
                    source = join
                to_connect.update(
                    {tc: (source, tc) for tc in checksums_to_connect})
            # Add sink node
            sink = pipeline.add(
                '{}_sink'.format(freq),
                RepositorySink(
                    (o.collection for o in outputs), pipeline),
                inputs=to_connect)
            # "De-iterate" (join) over iterators to get back to single child
            # node by the time we connect to the final node of the pipeline Set
            # the sink and subject_id as the default deiterator if there are no
            # deiterates (i.e. per_study) or to use as the upstream node to
            # connect the first deiterator for every frequency
            deiter_nodes[freq] = sink  # for per_study the "deiterator" == sink
            for iterator in sorted(pipeline.iterators(freq),
                                    key=deiter_node_sort_key):
                # Connect to previous deiterator or sink
                # NB: we only need to keep a reference to the last one in the
                # chain in order to connect with the "final" node, so we can
                # overwrite the entry in the 'deiter_nodes' dict
                deiter_nodes[freq] = pipeline.add(
                    '{}_{}_deiter'.format(freq, iterator),
                    IdentityInterface(
                        ['checksums']),
                    inputs={
                        'checksums': (deiter_nodes[freq], 'checksums')},
                    joinsource=iterator,
                    joinfield='checksums')
        # Create a final node, which is used to connect with downstream
        # pipelines
        pipeline.add(
            'final',
            Merge(
                len(deiter_nodes)),
            inputs={
                'in{}'.format(i): (di, 'checksums')
                for i, di in enumerate(deiter_nodes.values(), start=1)})


Instruction:
How would you explain what the following Python 3 function does
def _iterate(self, pipeline, to_process_array, subject_inds, visit_inds):
        # Check to see whether the subject/visit IDs to process (as specified
        # by the 'to_process' array) can be factorized into indepdent nodes,
        # i.e. all subjects to process have the same visits to process and
        # vice-versa.
        factorizable = True
        if len(list(pipeline.iterators())) == 2:
            nz_rows = to_process_array[to_process_array.any(axis=1), :]
            ref_row = nz_rows[0, :]
            factorizable = all((r == ref_row).all() for r in nz_rows)
        # If the subject/visit IDs to process cannot be factorized into
        # indepedent iterators, determine which to make make dependent on the
        # other in order to avoid/minimise duplicatation of download attempts
        dependent = None
        if not factorizable:
            input_freqs = list(pipeline.input_frequencies)
            # By default pick iterator the one with the most IDs to
            # iterate to be the dependent in order to reduce the number of
            # nodes created and any duplication of download attempts across
            # the nodes (if both 'per_visit' and 'per_subject' inputs are
            # required
            num_subjs, num_visits = nz_rows[:, nz_rows.any(axis=0)].shape
            if num_subjs > num_visits:
                dependent = self.study.SUBJECT_ID
            else:
                dependent = self.study.VISIT_ID
            if 'per_visit' in input_freqs:
                if 'per_subject' in input_freqs:
                    logger.warning(
                        "Cannot factorize sessions to process into independent"
                        " subject and visit iterators and both 'per_visit' and"
                        " 'per_subject' inputs are used by pipeline therefore"
                        " per_{} inputs may be cached twice".format(
                            dependent[:-3]))
                else:
                    dependent = self.study.SUBJECT_ID
            elif 'per_subject' in input_freqs:
                dependent = self.study.VISIT_ID
        # Invert the index dictionaries to get index-to-ID maps
        inv_subj_inds = {v: k for k, v in subject_inds.items()}
        inv_visit_inds = {v: k for k, v in visit_inds.items()}
        # Create iterator for subjects
        iter_nodes = {}
        if self.study.SUBJECT_ID in pipeline.iterators():
            fields = [self.study.SUBJECT_ID]
            if dependent == self.study.SUBJECT_ID:
                fields.append(self.study.VISIT_ID)
            # Add iterator node named after subject iterator
            subj_it = pipeline.add(self.study.SUBJECT_ID,
                                   IdentityInterface(fields))
            if dependent == self.study.SUBJECT_ID:
                # Subjects iterator is dependent on visit iterator (because of
                # non-factorizable IDs)
                subj_it.itersource = ('{}_{}'.format(pipeline.name,
                                                     self.study.VISIT_ID),
                                      self.study.VISIT_ID)
                subj_it.iterables = [(
                    self.study.SUBJECT_ID,
                    {inv_visit_inds[n]: [inv_subj_inds[m]
                                         for m in col.nonzero()[0]]
                     for n, col in enumerate(to_process_array.T)})]
            else:
                subj_it.iterables = (
                    self.study.SUBJECT_ID,
                    [inv_subj_inds[n]
                     for n in to_process_array.any(axis=1).nonzero()[0]])
            iter_nodes[self.study.SUBJECT_ID] = subj_it
        # Create iterator for visits
        if self.study.VISIT_ID in pipeline.iterators():
            fields = [self.study.VISIT_ID]
            if dependent == self.study.VISIT_ID:
                fields.append(self.study.SUBJECT_ID)
            # Add iterator node named after visit iterator
            visit_it = pipeline.add(self.study.VISIT_ID,
                                    IdentityInterface(fields))
            if dependent == self.study.VISIT_ID:
                visit_it.itersource = ('{}_{}'.format(pipeline.name,
                                                      self.study.SUBJECT_ID),
                                       self.study.SUBJECT_ID)
                visit_it.iterables = [(
                    self.study.VISIT_ID,
                    {inv_subj_inds[m]:[inv_visit_inds[n]
                                       for n in row.nonzero()[0]]
                     for m, row in enumerate(to_process_array)})]
            else:
                visit_it.iterables = (
                    self.study.VISIT_ID,
                    [inv_visit_inds[n]
                     for n in to_process_array.any(axis=0).nonzero()[0]])
            iter_nodes[self.study.VISIT_ID] = visit_it
        if dependent == self.study.SUBJECT_ID:
            pipeline.connect(visit_it, self.study.VISIT_ID,
                             subj_it, self.study.VISIT_ID)
        if dependent == self.study.VISIT_ID:
            pipeline.connect(subj_it, self.study.SUBJECT_ID,
                             visit_it, self.study.SUBJECT_ID)
        return iter_nodes

Response: Generate nodes that iterate over subjects and visits in the study that need to be processed by the pipeline.


Instruction:
Implement a function in Python 3 to
check whether the outputs of the pipeline are present in all sessions in the project repository and were generated with matching provenance. Return an 2D boolean array (subjects: rows, visits: cols) with the sessions to process marked True. Parameters ---------- pipeline : Pipeline The pipeline to determine the sessions to process required_ouputs : set[str] The names of the pipeline outputs that are required. If None all are deemed to be required prqs_to_process_array : 2-D numpy.array[bool] A two-dimensional boolean array, where rows and columns correspond correspond to subjects and visits in the repository tree. True values represent a subject/visit ID pairs that will be (re)processed in prerequisite pipelines and therefore need to be included in the returned array. to_skip_array : 2-D numpy.array[bool] Similar to prqs_to_process_array, except denote the subject/visits that are to be skipped due to missing inputs filter_array : 2-D numpy.array[bool] A two-dimensional boolean array, where rows and columns correspond correspond to subjects and visits in the repository tree. True values represent a subject/visit ID pairs to include in the current round of processing. Note that if the 'force' flag is not set, sessions won't be reprocessed unless the parameters and pipeline version saved in the provenance doesn't match that of the given pipeline. subject_inds : dict[str,int] Mapping from subject ID to index in filter|to_process arrays visit_inds : dict[str,int] Mapping from visit ID to index in filter|to_process arrays force : bool Whether to force reprocessing of all (filtered) sessions or not. Note that if 'force' is true we can't just return the filter array as it might be dilated by summary outputs (i.e. of frequency 'per_visit', 'per_subject' or 'per_study'). So we still loop through all outputs and treat them like they don't exist Returns ------- to_process_array : 2-D numpy.array[bool] A two-dimensional boolean array, where rows correspond to subjects and columns correspond to visits in the repository. True values represent subject/visit ID pairs to run the pipeline for

Response:
def _to_process(self, pipeline, required_outputs, prqs_to_process_array,
                    to_skip_array, filter_array, subject_inds, visit_inds,
                    force):
        """
        Check whether the outputs of the pipeline are present in all sessions
        in the project repository and were generated with matching provenance.
        Return an 2D boolean array (subjects: rows, visits: cols) with the
        sessions to process marked True.

        Parameters
        ----------
        pipeline : Pipeline
            The pipeline to determine the sessions to process
        required_ouputs : set[str]
            The names of the pipeline outputs that are required. If None all
            are deemed to be required
        prqs_to_process_array : 2-D numpy.array[bool]
            A two-dimensional boolean array, where rows and columns correspond
            correspond to subjects and visits in the repository tree. True
            values represent a subject/visit ID pairs that will be
            (re)processed in prerequisite pipelines and therefore need to be
            included in the returned array.
        to_skip_array : 2-D numpy.array[bool]
            Similar to prqs_to_process_array, except denote the subject/visits
            that are to be skipped due to missing inputs
        filter_array : 2-D numpy.array[bool]
            A two-dimensional boolean array, where rows and columns correspond
            correspond to subjects and visits in the repository tree. True
            values represent a subject/visit ID pairs to include
            in the current round of processing. Note that if the 'force'
            flag is not set, sessions won't be reprocessed unless the
            parameters and pipeline version saved in the provenance doesn't
            match that of the given pipeline.
        subject_inds : dict[str,int]
            Mapping from subject ID to index in filter|to_process arrays
        visit_inds : dict[str,int]
            Mapping from visit ID to index in filter|to_process arrays
        force : bool
            Whether to force reprocessing of all (filtered) sessions or not.
            Note that if 'force' is true we can't just return the filter array
            as it might be dilated by summary outputs (i.e. of frequency
            'per_visit', 'per_subject' or 'per_study'). So we still loop
            through all outputs and treat them like they don't exist

        Returns
        -------
        to_process_array : 2-D numpy.array[bool]
            A two-dimensional boolean array, where rows correspond to
            subjects and columns correspond to visits in the repository. True
            values represent subject/visit ID pairs to run the pipeline for
        """
        # Reference the study tree in local variable for convenience
        tree = self.study.tree
        # Check to see if the pipeline has any low frequency outputs, because
        # if not then each session can be processed indepdently. Otherwise,
        # the "session matrix" (as defined by subject_ids and visit_ids
        # passed to the Study class) needs to be complete, i.e. a session
        # exists (with the full complement of requird inputs) for each
        # subject/visit ID pair.
        summary_outputs = [
            o.name for o in pipeline.outputs if o.frequency != 'per_session']
        # Set of frequencies present in pipeline outputs
        output_freqs = pipeline.output_frequencies
        if summary_outputs:
            if list(tree.incomplete_subjects):
                raise ArcanaUsageError(
                    "Can't process '{}' pipeline as it has low frequency "
                    " outputs (i.e. outputs that aren't of 'per_session' "
                    "frequency) ({}) and subjects ({}) that are missing one "
                    "or more visits ({}). Please restrict the subject/visit "
                    "IDs in the study __init__ to continue the analysis"
                    .format(
                        self.name,
                        ', '.join(summary_outputs),
                        ', '.join(s.id for s in tree.incomplete_subjects),
                        ', '.join(v.id for v in tree.incomplete_visits)))

        def array_inds(x):
            """
            Get the array index corresponding to an item or node. If it doesn't
            have a subject and/or visit ID (i.e. frequency != 'per_session')
            then the first index in the row/column is selected as it will be
            dialated to the whole row/column later
            """
            return (subject_inds.get(x.subject_id, 0),
                    visit_inds.get(x.visit_id, 0))

        # Initalise array to represent which sessions need to be reprocessed
        to_process_array = np.zeros((len(subject_inds), len(visit_inds)),
                                    dtype=bool)
        # An array to mark outputs that have been altered outside of Arcana
        # and therefore protect from over-writing
        to_protect_array = np.copy(to_process_array)
        # Mark the sessions that we should check to see if the configuration
        # saved in the provenance record matches that of the current pipeline
        to_check_array = np.copy(to_process_array)
        # As well as the the sessions that need to be protected or skipped,
        # keep track of the items in those sessions that are protected or
        # missing for more informative error messages
        to_protect = defaultdict(list)
        to_skip = defaultdict(list)
        # Check data tree for missing inputs
        for input in pipeline.inputs:  # @ReservedAssignment
            # NB: Study inputs that don't have skip_missing set and have
            # missing data should raise an error before this point
            if input.skip_missing:
                for item in input.collection:
                    if not item.exists:
                        to_skip_array[array_inds(item)] = True
                        to_skip[array_inds(item)].append(item)
        # Dialate array over all iterators that are joined by the pipeline
        to_skip_array = self._dialate_array(to_skip_array, pipeline.joins)
        # Check data tree for missing required outputs
        for output in pipeline.outputs:
            # Check to see if output is required by downstream processing
            required = (required_outputs is None or
                        output.name in required_outputs)
            for item in output.collection:
                if item.exists:
                    # Check to see if checksums recorded when derivative
                    # was generated by previous run match those of current file
                    # set. If not we assume they have been manually altered and
                    # therefore should not be overridden
                    if item.checksums != item.recorded_checksums:
                        logger.warning(
                            "Checksums for {} do not match those recorded in "
                            "provenance. Assuming it has been manually "
                            "corrected outside of Arcana and will therefore "
                            "not overwrite. Please delete manually if this "
                            "is not intended".format(repr(item)))
                        to_protect_array[array_inds(item)] = True
                        to_protect[array_inds(item)].append(item)
                    elif required:
                        if force:
                            to_process_array[array_inds(item)] = True
                        else:
                            to_check_array[array_inds(item)] = True
                elif required:
                    to_process_array[array_inds(item)] = True
        # Filter sessions to process by those requested
        to_process_array *= filter_array
        to_check_array *= (filter_array * np.invert(to_process_array))
        if to_check_array.any() and self.prov_check:
            # Get list of sessions, subjects, visits, tree objects to check
            # their provenance against that of the pipeline
            to_check = []
            if 'per_session' in output_freqs:
                to_check.extend(
                    s for s in tree.sessions
                    if to_check_array[subject_inds[s.subject_id],
                                      visit_inds[s.visit_id]])
            if 'per_subject' in output_freqs:
                # We can just test the first col of outputs_exist as rows
                # should be either all True or all False
                to_check.extend(s for s in tree.subjects
                                if to_check_array[subject_inds[s.id], 0])
            if 'per_visit' in output_freqs:
                # We can just test the first row of outputs_exist as cols
                # should be either all True or all False
                to_check.extend(v for v in tree.visits
                                if to_check_array[0, visit_inds[v.id]])
            if 'per_study' in output_freqs:
                to_check.append(tree)
            for node in to_check:
                # Generated expected record from current pipeline/repository-
                # state
                requires_reprocess = False
                try:
                    # Retrieve record stored in tree node
                    record = node.record(pipeline.name, pipeline.study.name)
                    expected_record = pipeline.expected_record(node)

                    # Compare record with expected
                    mismatches = record.mismatches(expected_record,
                                                   self.prov_check,
                                                   self.prov_ignore)
                    if mismatches:
                        msg = ("mismatch in provenance:\n{}\n Add mismatching "
                               "paths (delimeted by '/') to 'prov_ignore' "
                               "argument of Processor to ignore"
                               .format(
                                   pformat(mismatches)))
                        requires_reprocess = True
                except ArcanaNameError as e:
                    msg = "missing provenance record"
                    requires_reprocess = False
                    to_protect_array[array_inds(node)] = True
                except ArcanaDataNotDerivedYetError as e:
                    msg = ("missing input '{}' and therefore cannot check "
                           "provenance".format(e.name))
                    requires_reprocess = True
                if requires_reprocess:
                    if self.reprocess:
                        to_process_array[array_inds(node)] = True
                        logger.info(
                            "Reprocessing {} with '{}' due to {}"
                            .format(node, pipeline.name, msg))
                    else:
                        raise ArcanaReprocessException(
                            "Cannot use derivatives for '{}' pipeline stored "
                            "in {} due to {}, set 'reprocess' "
                            "flag of Processor to overwrite".format(
                                pipeline.name, node, msg))

        def inds_to_ids(inds):
            subject_id = next(k for k, v in subject_inds.items()
                              if v == inds[0])
            visit_id = next(k for k, v in visit_inds.items() if v == inds[1])
            return (subject_id, visit_id)

        # Dialate to process array
        to_process_array = self._dialate_array(to_process_array,
                                               pipeline.joins)
        intersection = to_process_array * to_skip_array
        if intersection.any():
            missing_inputs_msg = ''
            missing_prq_inputs_msg = ''
            for sess_inds in zip(*np.nonzero(intersection)):
                subject_id, visit_id = inds_to_ids(sess_inds)
                if sess_inds in to_skip:
                    missing_inputs_msg += (
                        "\n(subject={}, visit={}): [{}]".format(
                            subject_id, visit_id,
                            ', '.join(i.name for i in to_skip[sess_inds])))
                else:
                    missing_prq_inputs_msg += (
                        "\n(subject={}, visit={})".format(subject_id,
                                                          visit_id))
            warning_msg = ("Skipping the following nodes for '{}' pipeline due"
                           " to ".format(pipeline))
            if missing_inputs_msg:
                warning_msg += "missing inputs:{}".format(missing_inputs_msg)
            if missing_prq_inputs_msg:
                if missing_inputs_msg:
                    warning_msg += "\nand the following due to "
                warning_msg += ("missing inputs to prerequisite pipelines:{}"
                                .format(missing_prq_inputs_msg))
            logger.warning(warning_msg)
        # Remove nodes that are to be skipped due to missing inputs
        to_process_array *= np.invert(to_skip_array)
        # Check for conflicts between nodes to process and nodes to protect
        conflicting = to_process_array * to_protect_array
        if conflicting.any():
            error_msg = ''
            for sess_inds in zip(*np.nonzero(conflicting)):
                subject_id, visit_id = inds_to_ids(sess_inds)
                if required_outputs is None:
                    conflict_outputs = pipeline.outputs
                else:
                    conflict_outputs = [pipeline.study.bound_spec(r)
                                        for r in required_outputs]
                items = [
                    o.collection.item(subject_id=subject_id, visit_id=visit_id)
                    for o in conflict_outputs]
                missing = [i for i in items if i not in to_protect[sess_inds]]
                error_msg += (
                    "\n({}, {}): protected=[{}], missing=[{}]"
                    .format(
                        subject_id, visit_id,
                        ', '.join(i.name for i in to_protect[sess_inds]),
                        ', '.join(i.name for i in missing)))
            raise ArcanaProtectedOutputConflictError(
                "Cannot process {} as there are nodes with both protected "
                "outputs (ones modified externally to Arcana) and missing "
                "required outputs. Either delete protected outputs or provide "
                "missing required outputs to continue:{}".format(pipeline,
                                                                 error_msg))
        # Add in any prerequisites to process that aren't explicitly protected
        to_process_array |= (prqs_to_process_array *
                             filter_array *
                             np.invert(to_protect_array))
        to_process_array = self._dialate_array(to_process_array,
                                               pipeline.joins)
        return to_process_array, to_protect_array, to_skip_array


Instruction:
Can you tell what is the following Python 3 function doing
def _dialate_array(self, array, iterators):
        if not iterators:
            return array
        dialated = np.copy(array)
        if self.study.SUBJECT_ID in iterators:
            # If we join over subjects we should include all subjects for every
            # visit we want to process
            dialated[:, dialated.any(axis=0)] = True
        if self.study.VISIT_ID in iterators:
            # If we join over visits we should include all visits for every
            # subject we want to process
            dialated[dialated.any(axis=1), :] = True
        return dialated

Response: Dialates a to_process array to include all subject and or visit for every unique entry in the set iterators.


Instruction:
Can you tell what is the following Python 3 function doing
def _get_args(self, node, keywords):
        args = super(ArcanaSlurmGraphPlugin, self)._get_args(
            node, keywords)
        # Substitute the template arg with the node-specific one
        new_args = []
        for name, arg in zip(keywords, args):
            if name == 'template':
                new_args.append(self._processor.slurm_template(node))
            else:
                new_args.append(arg)
        return tuple(new_args)

Response:
Intercept calls to get template and return our own node - specific oneCOOKIE - set - key - value pairs template
            - a node - id


Instruction:
Explain what the following Python 3 code does
def wall_time_str(self, wall_time):
        days = int(wall_time // 1440)
        hours = int((wall_time - days * 1440) // 60)
        minutes = int(math.floor(wall_time - days * 1440 - hours * 60))
        seconds = int((wall_time - math.floor(wall_time)) * 60)
        return "{}-{:0>2}:{:0>2}:{:0>2}".format(days, hours, minutes, seconds)

Response: Returns the wall time in the format required for the sbatch script.


Instruction:
Make a summary of the following Python 3 code
def sync(self):
        self._elk.add_handler('VN', self._vn_handler)
        self._elk.add_handler('XK', self._xk_handler)
        self._elk.add_handler('RP', self._rp_handler)
        self._elk.add_handler('IE', self._elk.call_sync_handlers)
        self._elk.add_handler('SS', self._ss_handler)
        self._elk.send(vn_encode())
        self._elk.send(lw_encode())
        self._elk.send(ss_encode())

Response: Retrieve panel information from ElkM1


Instruction:
Create a Python 3 function to
set the thermostat to the given value.

Response:
def set(self, element_to_set, value):
        """(Helper) Set thermostat"""
        self._elk.send(ts_encode(self.index, value, element_to_set))


Instruction:
Can you generate a brief explanation for the following Python 3 code
def renamed(self, name):
        duplicate = copy(self)
        duplicate._name = name
        return duplicate

Response: Duplicate the datum and rename it


Instruction:
Given the following Python 3 function, write the documentation
def compare(self, other):
        if self._req != other._req:
            raise ArcanaUsageError(
                "Can't compare versions of different requirements {} and {}"
                .format(self._req, other._req))
        # Compare main sequence
        if self._seq < other._seq:
            return -1
        elif self._seq > other._seq:
            return 1
        # If main sequence is equal check prerelease. If a prerelease is
        # None then it is a full release which is > then a prerelease so we
        # just assign it 'z' (which is greater than 'a', 'b' and 'rc')
        s = self._prerelease if self._prerelease is not None else ('z',)
        o = other._prerelease if other._prerelease is not None else ('z',)
        if s < o:
            return -1
        if s > o:
            return 1
        # If both main sequence and prereleases are equal, compare post release
        s = self._post if self._post is not None else 0
        o = other._post if other._post is not None else 0
        if s < o:
            return -1
        if s > o:
            return 1
        # If both main sequence and prereleases are equal, compare development
        # release
        s = self._dev if self._dev is not None else 0
        o = other._dev if other._dev is not None else 0
        if s < o:
            return -1
        if s > o:
            return 1
        assert self == other
        return 0

Response: Compares the version with another version.


Instruction:
Write a Python 3 script to
parse a typical version string into a tuple that can be used to pick the version number and the version number of the version.

Response:
def parse(self, version):
        """
        Splits a typical version string (e.g. <MAJOR>.<MINOR>.<MICRO>)
        into a tuple that can be sorted properly. Ignores all leading
        and trailing characters by using a regex search (instead of match) so
        as to pick the version string out of a block of text.

        Parameters
        ----------
        version : str | int | float | tuple(int)
            The string containing the version numbers, or alternatively an
            integer, float (number after decimal is interpreted as minor ver),
            or tuple|list containing the version sequence.

        Returns
        -------
        sequence : tuple(int | str)
            A tuple containing the main sequence of the version,
            e.g. <MAJOR>.<MINOR>.<MICRO>
        prerelease : 2-tuple(str, int) | None
            A 2-tuple containing the type of prerelease ('a' - alpha,
            'b' - beta, or 'rc' - release-canditate) and the number of the
            prerelease
        post : int |None
            The number of the post version
        dev : int | None
            The number of the development version
        """
        # Check to see if version is not a string but rather another type
        # that can be interpreted as a version
        if isinstance(version, int):
            return (version,), None, None, None
        elif isinstance(version, (tuple, list)):
            return tuple(int(i) for i in version), None, None, None
        elif isinstance(version, float):
            major = math.floor(version)
            minor = version - major
            return (major, minor), None, None, None
        match = self.regex.search(version)
        if match is None:
            raise ArcanaVersionNotDetectableError(
                "Could not parse version string {} as {}. Regex ({}) did not "
                "match any sub-string".format(version, type(self).__name__,
                                              self.regex.pattern))
        sequence = []
        prerelease = None
        dev = None
        post = None
        for part in match.group(1).split(self.delimeter):
            if part.startswith('dev'):
                dev = int(part[len('dev'):])
            elif part.startswith('post'):
                post = int(part[len('post'):])
            else:
                # Split on non-numeric parts of the version string so that we
                # can detect prerelease
                sub_parts = re.split('([^\d]+)', part)
                if sub_parts[0]:
                    try:
                        seq_part = int(sub_parts[0])
                    except ValueError:
                        seq_part = sub_parts[0]
                    sequence.append(seq_part)
                if len(sub_parts) > 1:
                    stage = sub_parts[1]
                    try:
                        pr_ver = int(sub_parts[2])
                    except ValueError:
                        raise ArcanaVersionNotDetectableError(
                            "Could not parse version string {} as {}. "
                            "Did not recognise pre-release version {}"
                            .format(version, type(self).__name__,
                                    sub_parts[2]))
                    stage = stage.strip('-_').lower()
                    if not stage:  # No prerelease info, assume a dev version
                        assert dev is None
                        dev = pr_ver
                        continue
                    if 'alpha'.startswith(stage):
                        stage = 'a'
                    elif 'beta'.startswith(stage):
                        stage = 'b'
                    elif stage == 'rc' or stage == 'release-canditate':
                        stage = 'rc'
                    else:
                        raise ArcanaVersionNotDetectableError(
                            "Could not parse version string {} as {}. "
                            "Did not recognise pre-release stage {}"
                            .format(version, type(self).__name__, stage))
                    prerelease = (stage, pr_ver)
        return tuple(sequence), prerelease, post, dev


Instruction:
Create a Python 3 function for
returning True if the current version is within the given version.

Response:
def within(self, version):
        """
        A single version can also be interpreted as an open range (i.e. no
        maximum version)
        """
        if not isinstance(version, Version):
            version = type(self._min_ver)(self._req, version)
        return version >= self


Instruction:
Here you have a function in Python 3, explain what it does
def v(self, version, max_version=None, **kwargs):
        if not isinstance(version, Version):
            version = self.version_cls(self, version, **kwargs)
        # Return a version range instead of version
        if max_version is not None:
            if not isinstance(max_version, Version):
                max_version = self.version_cls(self, max_version, **kwargs)
            version = VersionRange(version, max_version)
        return version

Response: Returns either a single version or a version range depending on whether two arguments are supplied or one.


Instruction:
Can you implement a function in Python 3 that
opens a connection to the database.

Response:
def open(self):
        '''Open a connection to the database.

        If a connection appears to be open already, transactions are committed
        and it is closed before proceeding. After establishing the connection,
        the searchIndex table is prepared (and dropped if it already exists).
        '''
        if self.conn is not None:
            self.close()

        self.conn = sqlite3.connect(self.filename)
        self.cursor = self.conn.cursor()

        c = self.cursor
        c.execute('SELECT name FROM sqlite_master WHERE type="table"')
        if (u'searchIndex',) in c:
            c.execute('DROP TABLE searchIndex')

            if self.verbose:
                print('Dropped existing table', file=sys.stderr)

        c.executescript(
            '''
            CREATE TABLE searchIndex
            (id INTEGER PRIMARY KEY, name TEXT, type TEXT, path TEXT);

            CREATE UNIQUE INDEX anchor ON searchIndex (name, type, path);
            '''
        )


Instruction:
Given the following Python 3 function, write the documentation
def insert(self, name, entry_type, filename):
        '''Insert an entry into the Zeal database.

        Args:
            name: A string representing the name of the entry.
            entry_type: A string representing the entry type.
            filename: A string representing the filename of the documentation
                for the entry.

        Raises:
            RuntimeError: a database connection was not established before
                calling insert()
        '''
        if self.cursor is None:
            raise RuntimeError(
                'Open DB connection before attempting to call insert!')

        db_entry = (name, entry_type, filename)

        if self.verbose:
            print('Inserting %s "%s" -> %s' % db_entry, file=sys.stderr)

        self.cursor.execute(
            '''INSERT OR IGNORE INTO searchIndex(name, type, path)
               VALUES (?, ?, ?)''', db_entry)

Response: Insert an entry into the Zeal database.


Instruction:
Can you generate the documentation for the following Python 3 function
def data(self, name, subject_ids=None, visit_ids=None, session_ids=None,
             **kwargs):
        """
        Returns the Fileset(s) or Field(s) associated with the provided spec
        name(s), generating derived filesets as required. Multiple names in a
        list can be provided, to allow their workflows to be combined into a
        single workflow.

        Parameters
        ----------
        name : str | List[str]
            The name of the FilesetSpec|FieldSpec to retried the
            filesets for
        subject_id : str | None
            The subject ID of the data to return. If provided (including None
            values) the data will be return as a single item instead of a
            collection
        visit_id : str | None
            The visit ID of the data to return. If provided (including None
            values) the data will be return as a single item instead of a
            c ollection
        subject_ids : list[str]
            The subject IDs to include in the returned collection
        visit_ids : list[str]
            The visit IDs to include in the returned collection
        session_ids : list[str]
            The session IDs (i.e. 2-tuples of the form
            (<subject-id>, <visit-id>) to include in the returned collection

        Returns
        -------
        data : BaseItem | BaseCollection | list[BaseItem | BaseCollection]
            If 'subject_id' or 'visit_id' is provided then the data returned is
            a single Fileset or Field. Otherwise a collection of Filesets or
            Fields are returned. If muliple spec names are provided then a
            list of items or collections corresponding to each spec name.
        """
        if isinstance(name, basestring):
            single_name = True
            names = [name]
        else:
            names = name
            single_name = False
        single_item = 'subject_id' in kwargs or 'visit_id' in kwargs
        filter_items = (subject_ids, visit_ids, session_ids) != (None, None,
                                                                   None)
        specs = [self.spec(n) for n in names]
        if single_item:
            if filter_items:
                raise ArcanaUsageError(
                    "Cannot provide 'subject_id' and/or 'visit_id' in "
                    "combination with 'subject_ids', 'visit_ids' or "
                    "'session_ids'")
            subject_id = kwargs.pop('subject_id', None)
            visit_id = kwargs.pop('visit_id', None)
            iterators = set(chain(self.FREQUENCIES[s.frequency]
                                  for s in specs))
            if subject_id is not None and visit_id is not None:
                session_ids = [(subject_id, visit_id)]
            elif subject_id is not None:
                if self.VISIT_ID in iterators:
                    raise ArcanaUsageError(
                        "Non-None values for visit IDs need to be "
                        "provided to select a single item for each of '{}'"
                        .format("', '".join(names)))
                subject_ids = [subject_id]
            elif visit_id is not None:
                if self.SUBJECT_ID in iterators:
                    raise ArcanaUsageError(
                        "Non-None values for subject IDs need to be "
                        "provided to select a single item for each of '{}'"
                        .format("', '".join(names)))
                visit_ids = [visit_id]
            elif iterators:
                raise ArcanaUsageError(
                    "Non-None values for subject and/or visit IDs need to be "
                    "provided to select a single item for each of '{}'"
                    .format("', '".join(names)))
        # Work out which pipelines need to be run
        pipeline_getters = defaultdict(set)
        for spec in specs:
            if spec.derived or spec.derivable:  # Filter out Study inputs
                # Add name of spec to set of required outputs
                pipeline_getters[spec.pipeline_getter].add(spec.name)
        # Run required pipelines
        if pipeline_getters:
            kwargs = copy(kwargs)
            kwargs.update({'subject_ids': subject_ids,
                           'visit_ids': visit_ids,
                           'session_ids': session_ids})
            pipelines, required_outputs = zip(*(
                (self.pipeline(k), v) for k, v in pipeline_getters.items()))
            kwargs['required_outputs'] = required_outputs
            self.processor.run(*pipelines, **kwargs)
        # Find and return Item/Collection corresponding to requested spec
        # names
        all_data = []
        for name in names:
            spec = self.bound_spec(name)
            data = spec.collection
            if single_item:
                data = data.item(subject_id=subject_id, visit_id=visit_id)
            elif filter_items and spec.frequency != 'per_study':
                if subject_ids is None:
                    subject_ids = []
                if visit_ids is None:
                    visit_ids = []
                if session_ids is None:
                    session_ids = []
                if spec.frequency == 'per_session':
                    data = [d for d in data
                            if (d.subject_id in subject_ids or
                                d.visit_id in visit_ids or
                                d.session_id in session_ids)]
                elif spec.frequency == 'per_subject':
                    data = [d for d in data
                            if (d.subject_id in subject_ids or
                                d.subject_id in [s[0] for s in session_ids])]
                elif spec.frequency == 'per_visit':
                    data = [d for d in data
                            if (d.visit_id in visit_ids or
                                d.visit_id in [s[1] for s in session_ids])]
                if not data:
                    raise ArcanaUsageError(
                        "No matching data found (subject_ids={}, visit_ids={} "
                        ", session_ids={})"
                        .format(subject_ids, visit_ids, session_ids))
                data = spec.CollectionClass(spec.name, data)
            if single_name:
                return data
            else:
                all_data.append(data)
        return all_data

Response: Returns the data for the given spec - related fileset or Field.


Instruction:
Can you implement a function in Python 3 that
checks whether the given switch matches the given value.

Response:
def branch(self, name, values=None):  # @UnusedVariable @IgnorePep8
        """
        Checks whether the given switch matches the value provided

        Parameters
        ----------
        name : str
            The name of the parameter to retrieve
        value : str | None
            The value(s) of the switch to match if a non-boolean switch
        """
        if isinstance(values, basestring):
            values = [values]
        spec = self.parameter_spec(name)
        if not isinstance(spec, SwitchSpec):
            raise ArcanaUsageError(
                "{} is standard parameter not a switch".format(spec))
        switch = self._get_parameter(name)
        if spec.is_boolean:
            if values is not None:
                raise ArcanaDesignError(
                    "Should not provide values ({}) to boolean switch "
                    "'{}' in {}".format(
                        values, name, self._param_error_location))
            in_branch = switch.value
        else:
            if values is None:
                raise ArcanaDesignError(
                    "Value(s) need(s) to be provided non-boolean switch"
                    " '{}' in {}".format(
                        name, self._param_error_location))
            # Register parameter as being used by the pipeline
            unrecognised_values = set(values) - set(spec.choices)
            if unrecognised_values:
                raise ArcanaDesignError(
                    "Provided value(s) ('{}') for switch '{}' in {} "
                    "is not a valid option ('{}')".format(
                        "', '".join(unrecognised_values), name,
                        self._param_error_location,
                        "', '".join(spec.choices)))
            in_branch = switch.value in values
        return in_branch


Instruction:
Explain what the following Python 3 code does
def unhandled_branch(self, name):
        raise ArcanaDesignError(
            "'{}' value of '{}' switch in {} is not handled"
            .format(self._get_parameter(name), name,
                    self._param_error_location))

Response: This method raises an exception if a pipeline doesn t handle a particular switch value


Instruction:
Make a summary of the following Python 3 code
def save_workflow_graph_for(self, spec_name, fname, full=False,
                                style='flat', **kwargs):
        """
        Saves a graph of the workflow to generate the requested spec_name

        Parameters
        ----------
        spec_name : str
            Name of the spec to generate the graph for
        fname : str
            The filename for the saved graph
        style : str
            The style of the graph, can be one of can be one of
            'orig', 'flat', 'exec', 'hierarchical'
        """
        pipeline = self.spec(spec_name).pipeline
        if full:
            workflow = pe.Workflow(name='{}_gen'.format(spec_name),
                                   base_dir=self.processor.work_dir)
            self.processor._connect_pipeline(
                pipeline, workflow, **kwargs)
        else:
            workflow = pipeline._workflow
        fname = op.expanduser(fname)
        if not fname.endswith('.png'):
            fname += '.png'
        dotfilename = fname[:-4] + '.dot'
        workflow.write_graph(graph2use=style,
                             dotfilename=dotfilename)

Response: Saves a graph of the workflow for the given spec_name to fname.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def spec(self, name):
        # If the provided "name" is actually a data item or parameter then
        # replace it with its name.
        if isinstance(name, (BaseData, Parameter)):
            name = name.name
        # If name is a parameter than return the parameter spec
        if name in self._param_specs:
            return self._param_specs[name]
        else:
            return self.bound_spec(name)

Response: Returns either the input corresponding to a fileset or field or a spec or a parameter that has either been passed to the study as an input or can be derived.


Instruction:
Write a Python 3 script to
return an input selector or derived spec bound to the study i. e. the input selector or derived spec bound to the study i. e. the repository tree is checked for existing outputs

Response:
def bound_spec(self, name):
        """
        Returns an input selector or derived spec bound to the study, i.e.
        where the repository tree is checked for existing outputs

        Parameters
        ----------
        name : Str
            A name of a fileset or field
        """
        # If the provided "name" is actually a data item or parameter then
        # replace it with its name.
        if isinstance(name, BaseData):
            name = name.name
        # Get the spec from the class
        spec = self.data_spec(name)
        try:
            bound = self._inputs[name]
        except KeyError:
            if not spec.derived and spec.default is None:
                raise ArcanaMissingDataException(
                    "Acquired (i.e. non-generated) fileset '{}' "
                    "was not supplied when the study '{}' was "
                    "initiated".format(name, self.name))
            else:
                try:
                    bound = self._bound_specs[name]
                except KeyError:
                    bound = self._bound_specs[name] = spec.bind(self)
        return bound


Instruction:
Explain what the following Python 3 code does
def data_spec(cls, name):
        # If the provided "name" is actually a data item or parameter then
        # replace it with its name.
        if isinstance(name, BaseData):
            name = name.name
        try:
            return cls._data_specs[name]
        except KeyError:
            raise ArcanaNameError(
                name,
                "No fileset spec named '{}' in {}, available:\n{}"
                .format(name, cls.__name__,
                        "\n".join(list(cls._data_specs.keys()))))

Response:
Returns the fileset_spec i. e. the template of the fileset expected to
        be supplied or generated corresponding to the fileset_spec name.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def cache_inputs(self):
        workflow = pe.Workflow(name='cache_download',
                               base_dir=self.processor.work_dir)
        subjects = pe.Node(IdentityInterface(['subject_id']), name='subjects',
                           environment=self.environment)
        sessions = pe.Node(IdentityInterface(['subject_id', 'visit_id']),
                           name='sessions', environment=self.environment)
        subjects.iterables = ('subject_id', tuple(self.subject_ids))
        sessions.iterables = ('visit_id', tuple(self.visit_ids))
        source = pe.Node(RepositorySource(
            self.bound_spec(i).collection for i in self.inputs), name='source')
        workflow.connect(subjects, 'subject_id', sessions, 'subject_id')
        workflow.connect(sessions, 'subject_id', source, 'subject_id')
        workflow.connect(sessions, 'visit_id', source, 'visit_id')
        workflow.run()

Response:
Runs the Study s repository source node for each of the inputs
        of the study.


Instruction:
Can you generate the documentation for the following Python 3 function
def provided(self, spec_name, default_okay=True):
        try:
            spec = self.bound_spec(spec_name)
        except ArcanaMissingDataException:
            return False
        if isinstance(spec, BaseInputSpec):
            return spec.default is not None and default_okay
        else:
            return True

Response: Checks to see if the corresponding data spec was provided an explicit input.


Instruction:
Implement a function in Python 3 to
return the frequency corresponding to the given iterators

Response:
def freq_from_iterators(cls, iterators):
        """
        Returns the frequency corresponding to the given iterators
        """
        return {
            set(it): f for f, it in cls.FREQUENCIES.items()}[set(iterators)]