Instruction:
Can you generate the documentation for the following Python 3 function
def getSolrType(self, field):
        ftype = 'string'
        try:
            ftype = self.fieldtypes[field]
            return ftype
        except Exception:
            pass
        fta = field.split('_')
        if len(fta) > 1:
            ft = fta[len(fta) - 1]
            try:
                ftype = self.fieldtypes[ft]
                # cache the type so it's used next time
                self.fieldtypes[field] = ftype
            except Exception:
                pass
        return ftype

Response: Returns the SOLR type of the specified field name.


Instruction:
Write a Python 3 function for
returning the number of entries that match query.

Response:
def count(self, q='*:*', fq=None):
        """Return the number of entries that match query."""
        params = {'q': q, 'rows': '0'}
        if fq is not None:
            params['fq'] = fq
        res = self.search(params)
        hits = res['response']['numFound']
        return hits


Instruction:
Write a Python 3 script for
returning a dictionary of ids and values for the current index.

Response:
def getIds(self, query='*:*', fq=None, start=0, rows=1000):
        """Returns a dictionary of: matches: number of matches failed: if true, then an
        exception was thrown start: starting index ids: [id, id, ...]

        See also the SOLRSearchResponseIterator class

        """
        params = {'q': query, 'start': str(start), 'rows': str(rows), 'wt': 'python'}
        if fq is not None:
            params['fq'] = fq
        request = urllib.parse.urlencode(params, doseq=True)
        data = None
        response = {'matches': 0, 'start': start, 'failed': True, 'ids': []}
        try:
            rsp = self.doPost(self.solrBase + '', request, self.formheaders)
            data = eval(rsp.read())
        except Exception:
            pass
        if data is None:
            return response
        response['failed'] = False
        response['matches'] = data['response']['numFound']
        for doc in data['response']['docs']:
            response['ids'].append(doc['id'][0])
        return response


Instruction:
Implement a function in Python 3 to
retrieve the specified document.

Response:
def get(self, id):
        """Retrieves the specified document."""
        params = {'q': 'id:%s' % str(id), 'wt': 'python'}
        request = urllib.parse.urlencode(params, doseq=True)
        data = None
        try:
            rsp = self.doPost(self.solrBase + '', request, self.formheaders)
            data = eval(rsp.read())
        except Exception:
            pass
        if data['response']['numFound'] > 0:
            return data['response']['docs'][0]
        return None


Instruction:
Given the following Python 3 function, write the documentation
def getFields(self, numTerms=1):
        if self._fields is not None:
            return self._fields
        params = {'numTerms': str(numTerms), 'wt': 'python'}
        request = urllib.parse.urlencode(params, doseq=True)
        rsp = self.doPost(self.solrBase + '/admin/luke', request, self.formheaders)
        data = eval(rsp.read())
        self._fields = data
        return data

Response: Retrieve a list of fields.


Instruction:
Write a Python 3 script for
retrieving the unique values for a field along with their usage counts.

Response:
def fieldValues(self, name, q="*:*", fq=None, maxvalues=-1):
        """Retrieve the unique values for a field, along with their usage counts.
        http://localhost:8080/solr/select/?q=*:*&rows=0&facet=true&inde
        nt=on&wt=python&facet.field=genus_s&facet.limit=10&facet.zeros=false&fa
        cet.sort=false.

        @param name(string) Name of field to retrieve values for
        @param q(string) Query identifying the records from which values will be retrieved
        @param fq(string) Filter query restricting operation of query
        @param maxvalues(int) Maximum number of values to retrieve.  Default is -1,
          which causes retrieval of all values.
        @return dict of {fieldname: [[value, count], ... ], }

        """
        params = {
            'q': q,
            'rows': '0',
            'facet': 'true',
            'facet.field': name,
            'facet.limit': str(maxvalues),
            'facet.zeros': 'false',
            'wt': 'python',
            'facet.sort': 'false',
        }
        if fq is not None:
            params['fq'] = fq
        request = urllib.parse.urlencode(params, doseq=True)
        rsp = self.doPost(self.solrBase + '', request, self.formheaders)
        data = eval(rsp.read())
        return data['facet_counts']['facet_fields']


Instruction:
Write a Python 3 script to
return the python type of the specified field name.

Response:
def getftype(self, name):
        """Returns the python type for the specified field name.  The field list is
        cached so multiple calls do not invoke a getFields request each time.

        @param name(string) The name of the SOLR field
        @returns Python type of the field.

        """
        fields = self.getFields()
        try:
            fld = fields['fields'][name]
        except Exception:
            return str
        if fld['type'] in ['string', 'text', 'stext', 'text_ws']:
            return str
        if fld['type'] in ['sint', 'integer', 'long', 'slong']:
            return int
        if fld['type'] in ['sdouble', 'double', 'sfloat', 'float']:
            return float
        if fld['type'] in ['boolean']:
            return bool
        return fld['type']


Instruction:
Create a Python 3 function to
generate a histogram of values from a string field.

Response:
def fieldAlphaHistogram(
        self, name, q='*:*', fq=None, nbins=10, includequeries=True
    ):
        """Generates a histogram of values from a string field. Output is:

        [[low, high, count, query],    ... ] Bin edges is determined by equal division
        of the fields

        """
        oldpersist = self.persistent
        self.persistent = True
        bins = []
        qbin = []
        fvals = []
        try:
            # get total number of values for the field
            # TODO: this is a slow mechanism to retrieve the number of distinct values
            # Need to replace this with something more efficient.
            ## Can probably replace with a range of alpha chars - need to check on
            ## case sensitivity
            fvals = self.fieldValues(name, q, fq, maxvalues=-1)
            nvalues = len(fvals[name]) / 2
            if nvalues < nbins:
                nbins = nvalues
            if nvalues == nbins:
                # Use equivalence instead of range queries to retrieve the values
                for i in range(0, nbins):
                    bin = [fvals[name][i * 2], fvals[name][i * 2], 0]
                    binq = '%s:%s' % (name, self.prepareQueryTerm(name, bin[0]))
                    qbin.append(binq)
                    bins.append(bin)
            else:
                delta = nvalues / nbins
                if delta == 1:
                    # Use equivalence queries, except the last one which includes the
                    # remainder of terms
                    for i in range(0, nbins - 2):
                        bin = [fvals[name][i * 2], fvals[name][i * 2], 0]
                        binq = '%s:%s' % (name, self.prepareQueryTerm(name, bin[0]))
                        qbin.append(binq)
                        bins.append(bin)
                    term = fvals[name][(nbins - 1) * 2]
                    bin = [term, fvals[name][((nvalues - 1) * 2)], 0]
                    binq = '%s:[%s TO *]' % (name, self.prepareQueryTerm(name, term))
                    qbin.append(binq)
                    bins.append(bin)
                else:
                    # Use range for all terms
                    # now need to page through all the values and get those at the edges
                    coffset = 0.0
                    delta = float(nvalues) / float(nbins)
                    for i in range(0, nbins):
                        idxl = int(coffset) * 2
                        idxu = (int(coffset + delta) * 2) - 2
                        bin = [fvals[name][idxl], fvals[name][idxu], 0]
                        # logging.info(str(bin))
                        binq = ''
                        try:
                            if i == 0:
                                binq = '%s:[* TO %s]' % (
                                    name,
                                    self.prepareQueryTerm(name, bin[1]),
                                )
                            elif i == nbins - 1:
                                binq = '%s:[%s TO *]' % (
                                    name,
                                    self.prepareQueryTerm(name, bin[0]),
                                )
                            else:
                                binq = '%s:[%s TO %s]' % (
                                    name,
                                    self.prepareQueryTerm(name, bin[0]),
                                    self.prepareQueryTerm(name, bin[1]),
                                )
                        except Exception:
                            self.logger.exception('Exception 1 in fieldAlphaHistogram:')
                        qbin.append(binq)
                        bins.append(bin)
                        coffset = coffset + delta
            # now execute the facet query request
            params = {
                'q': q,
                'rows': '0',
                'facet': 'true',
                'facet.field': name,
                'facet.limit': '1',
                'facet.mincount': 1,
                'wt': 'python',
            }
            request = urllib.parse.urlencode(params, doseq=True)
            for sq in qbin:
                try:
                    request = request + '&%s' % urllib.parse.urlencode(
                        {'facet.query': self.encoder(sq)[0]}
                    )
                except Exception:
                    self.logger.exception('Exception 2 in fieldAlphaHistogram')
            rsp = self.doPost(self.solrBase + '', request, self.formheaders)
            data = eval(rsp.read())
            for i in range(0, len(bins)):
                v = data['facet_counts']['facet_queries'][qbin[i]]
                bins[i][2] = v
                if includequeries:
                    bins[i].append(qbin[i])
        finally:
            self.persistent = oldpersist
            if not self.persistent:
                self.conn.close()
        return bins


Instruction:
Explain what the following Python 3 code does
def fieldHistogram(
        self, name, q="*:*", fq=None, nbins=10, minmax=None, includequeries=True
    ):
        """Generates a histogram of values. Expects the field to be integer or floating
        point.

        @param name(string) Name of the field to compute
        @param q(string) The query identifying the set of records for the histogram
        @param fq(string) Filter query to restrict application of query
        @param nbins(int) Number of bins in resulting histogram

        @return list of [binmin, binmax, n, binquery]

        """
        oldpersist = self.persistent
        self.persistent = True
        ftype = self.getftype(name)
        if ftype == str:
            ##handle text histograms over here
            bins = self.fieldAlphaHistogram(
                name, q=q, fq=fq, nbins=nbins, includequeries=includequeries
            )
            self.persistent = oldpersist
            if not self.persistent:
                self.conn.close()
            return bins
        bins = []
        qbin = []
        fvals = self.fieldValues(name, q, fq, maxvalues=nbins + 1)
        if len(fvals[name]) < 3:
            return bins
        nvalues = len(fvals[name]) / 2
        if nvalues < nbins:
            nbins = nvalues
        minoffset = 1
        if ftype == float:
            minoffset = 0.00001
        try:
            if minmax is None:
                minmax = self.fieldMinMax(name, q=q, fq=fq)
                # logging.info("MINMAX = %s" % str(minmax))
                minmax[0] = float(minmax[0])
                minmax[1] = float(minmax[1])
            delta = (minmax[1] - minmax[0]) / nbins
            for i in range(0, nbins):
                binmin = minmax[0] + (i * delta)
                bin = [binmin, binmin + delta, 0]
                if ftype == int:
                    bin[0] = int(bin[0])
                    bin[1] = int(bin[1])
                    if i == 0:
                        binq = '%s:[* TO %d]' % (name, bin[1])
                    elif i == nbins - 1:
                        binq = '%s:[%d TO *]' % (name, bin[0] + minoffset)
                        bin[0] = bin[0] + minoffset
                        if bin[1] < bin[0]:
                            bin[1] = bin[0]
                    else:
                        binq = '%s:[%d TO %d]' % (name, bin[0] + minoffset, bin[1])
                        bin[0] = bin[0] + minoffset
                else:
                    if i == 0:
                        binq = '%s:[* TO %f]' % (name, bin[1])
                    elif i == nbins - 1:
                        binq = '%s:[%f TO *]' % (name, bin[0] + minoffset)
                    else:
                        binq = '%s:[%f TO %f]' % (name, bin[0] + minoffset, bin[1])
                qbin.append(binq)
                bins.append(bin)

            # now execute the facet query request
            params = {
                'q': q,
                'rows': '0',
                'facet': 'true',
                'facet.field': name,
                'facet.limit': '1',
                'facet.mincount': 1,
                'wt': 'python',
            }
            request = urllib.parse.urlencode(params, doseq=True)
            for sq in qbin:
                request = request + '&%s' % urllib.parse.urlencode({'facet.query': sq})
            rsp = self.doPost(self.solrBase + '', request, self.formheaders)
            data = eval(rsp.read())
            for i in range(0, len(bins)):
                v = data['facet_counts']['facet_queries'][qbin[i]]
                bins[i][2] = v
                if includequeries:
                    bins[i].append(qbin[i])
        finally:
            self.persistent = oldpersist
            if not self.persistent:
                self.conn.close()
        return bins

Response: Generates a histogram of values for a single field.


Instruction:
Explain what the following Python 3 code does
def _nextPage(self, offset):
        self.logger.debug("Iterator crecord=%s" % str(self.crecord))
        params = {
            'q': self.q,
            'start': str(offset),
            'rows': str(self.pagesize),
            'fl': self.fields,
            'explainOther': '',
            'hl.fl': '',
        }
        if self.fq is not None:
            params['fq'] = self.fq
        self.res = self.client.search(params)
        self._numhits = int(self.res['response']['numFound'])

Response: Retrieves the next set of results from the service.


Instruction:
How would you explain what the following Python 3 function does
def _nextPage(self, offset):
        self.logger.debug("Iterator crecord=%s" % str(self.crecord))

        params = {
            'q': self.q,
            'rows': '0',
            'facet': 'true',
            'facet.field': self.field,
            'facet.limit': str(self.pagesize),
            'facet.offset': str(offset),
            'facet.zeros': 'false',
            'wt': 'python',
        }
        if self.fq is not None:
            params['fq'] = self.fq
        request = urllib.parse.urlencode(params, doseq=True)
        rsp = self.client.doPost(
            self.client.solrBase + '', request, self.client.formheaders
        )
        data = eval(rsp.read())
        try:
            self.res = data['facet_counts']['facet_fields'][self.field]
            self.logger.debug(self.res)
        except Exception:
            self.res = []
        self.index = 0

Response: Retrieves the next set of results from the service.


Instruction:
Can you generate the documentation for the following Python 3 function
def get_log(request):
    query = d1_gmn.app.models.EventLog.objects.all().order_by('timestamp', 'id')
    if not d1_gmn.app.auth.is_trusted_subject(request):
        query = d1_gmn.app.db_filter.add_access_policy_filter(
            request, query, 'sciobj__id'
        )
        query = d1_gmn.app.db_filter.add_redact_annotation(request, query)
    query = d1_gmn.app.db_filter.add_datetime_filter(
        request, query, 'timestamp', 'fromDate', 'gte'
    )
    query = d1_gmn.app.db_filter.add_datetime_filter(
        request, query, 'timestamp', 'toDate', 'lt'
    )
    query = d1_gmn.app.db_filter.add_string_filter(
        request, query, 'event__event', 'event'
    )
    if d1_gmn.app.views.util.is_v1_api(request):
        query = d1_gmn.app.db_filter.add_string_begins_with_filter(
            request, query, 'sciobj__pid__did', 'pidFilter'
        )
    elif d1_gmn.app.views.util.is_v2_api(request):
        query = d1_gmn.app.db_filter.add_sid_or_string_begins_with_filter(
            request, query, 'sciobj__pid__did', 'idFilter'
        )
    else:
        assert False, 'Unable to determine API version'
    total_int = query.count()
    query, start, count = d1_gmn.app.views.slice.add_slice_filter(
        request, query, total_int
    )
    return {
        'query': query,
        'start': start,
        'count': count,
        'total': total_int,
        'type': 'log',
    }

Response: Get log records from the log table.


Instruction:
Here you have a function in Python 3, explain what it does
def get_node(request):
    api_major_int = 2 if d1_gmn.app.views.util.is_v2_api(request) else 1
    node_pretty_xml = d1_gmn.app.node.get_pretty_xml(api_major_int)
    return django.http.HttpResponse(node_pretty_xml, d1_common.const.CONTENT_TYPE_XML)

Response: Return a pretty - printed node.


Instruction:
Here you have a function in Python 3, explain what it does
def get_replica(request, pid):
    _assert_node_is_authorized(request, pid)
    sciobj = d1_gmn.app.models.ScienceObject.objects.get(pid__did=pid)
    content_type_str = d1_gmn.app.object_format_cache.get_content_type(
        sciobj.format.format
    )
    # Replica is always a local file that can be handled with FileResponse()
    response = django.http.FileResponse(
        d1_gmn.app.sciobj_store.open_sciobj_file_by_pid(pid), content_type_str
    )
    d1_gmn.app.views.headers.add_sciobj_properties_headers_to_response(response, sciobj)
    # Log the replication of this object.
    d1_gmn.app.event_log.log_replicate_event(pid, request)
    return response

Response: MNReplication. getReplica → OctetStream.


Instruction:
Here you have a function in Python 3, explain what it does
def post_refresh_system_metadata(request):
    d1_gmn.app.views.assert_db.post_has_mime_parts(
        request,
        (
            ('field', 'pid'),
            ('field', 'serialVersion'),
            ('field', 'dateSysMetaLastModified'),
        ),
    )
    d1_gmn.app.views.assert_db.is_existing_object(request.POST['pid'])
    d1_gmn.app.models.sysmeta_refresh_queue(
        request.POST['pid'],
        request.POST['serialVersion'],
        request.POST['dateSysMetaLastModified'],
        'queued',
    ).save()
    return d1_gmn.app.views.util.http_response_with_boolean_true_type()

Response: MNStorage. systemMetadataChanged → boolean.


Instruction:
Explain what the following Python 3 code does
def post_object_list(request):
    d1_gmn.app.views.assert_db.post_has_mime_parts(
        request, (('field', 'pid'), ('file', 'object'), ('file', 'sysmeta'))
    )
    url_pid = request.POST['pid']
    sysmeta_pyxb = d1_gmn.app.sysmeta.deserialize(request.FILES['sysmeta'])
    d1_gmn.app.views.assert_sysmeta.obsoletes_not_specified(sysmeta_pyxb)
    d1_gmn.app.views.assert_sysmeta.matches_url_pid(sysmeta_pyxb, url_pid)
    d1_gmn.app.views.assert_sysmeta.is_valid_sid_for_new_standalone(sysmeta_pyxb)
    d1_gmn.app.views.create.create_sciobj(request, sysmeta_pyxb)
    return url_pid

Response: MNStorage. create method for POST requests.


Instruction:
Explain what the following Python 3 code does
def put_object(request, old_pid):
    if django.conf.settings.REQUIRE_WHITELIST_FOR_UPDATE:
        d1_gmn.app.auth.assert_create_update_delete_permission(request)
    d1_gmn.app.util.coerce_put_post(request)
    d1_gmn.app.views.assert_db.post_has_mime_parts(
        request, (('field', 'newPid'), ('file', 'object'), ('file', 'sysmeta'))
    )
    d1_gmn.app.views.assert_db.is_valid_pid_to_be_updated(old_pid)
    sysmeta_pyxb = d1_gmn.app.sysmeta.deserialize(request.FILES['sysmeta'])
    new_pid = request.POST['newPid']
    d1_gmn.app.views.assert_sysmeta.matches_url_pid(sysmeta_pyxb, new_pid)
    d1_gmn.app.views.assert_sysmeta.obsoletes_matches_pid_if_specified(
        sysmeta_pyxb, old_pid
    )
    sysmeta_pyxb.obsoletes = old_pid
    sid = d1_common.xml.get_opt_val(sysmeta_pyxb, 'seriesId')
    d1_gmn.app.views.assert_sysmeta.is_valid_sid_for_chain(old_pid, sid)
    d1_gmn.app.views.create.create_sciobj(request, sysmeta_pyxb)
    # The create event for the new object is added in create_sciobj(). The update
    # event on the old object is added here.
    d1_gmn.app.event_log.log_update_event(
        old_pid,
        request,
        timestamp=d1_common.date_time.normalize_datetime_to_utc(
            sysmeta_pyxb.dateUploaded
        ),
    )
    d1_gmn.app.sysmeta.update_modified_timestamp(old_pid)
    return new_pid

Response: MNStorage.update(session, pid, object, newPid, sysmeta) → Identifier.


Instruction:
Explain what the following Python 3 code does
def post_generate_identifier(request):
    d1_gmn.app.views.assert_db.post_has_mime_parts(request, (('field', 'scheme'),))
    if request.POST['scheme'] != 'UUID':
        raise d1_common.types.exceptions.InvalidRequest(
            0, 'Only the UUID scheme is currently supported'
        )
    fragment = request.POST.get('fragment', None)
    while True:
        pid = (fragment if fragment else '') + uuid.uuid4().hex
        if not d1_gmn.app.models.ScienceObject.objects.filter(pid__did=pid).exists():
            return pid

Response: Generate identifier for the current post.


Instruction:
Explain what the following Python 3 code does
def post_replicate(request):
    d1_gmn.app.views.assert_db.post_has_mime_parts(
        request, (('field', 'sourceNode'), ('file', 'sysmeta'))
    )
    sysmeta_pyxb = d1_gmn.app.sysmeta.deserialize(request.FILES['sysmeta'])
    d1_gmn.app.local_replica.assert_request_complies_with_replication_policy(
        sysmeta_pyxb
    )
    pid = d1_common.xml.get_req_val(sysmeta_pyxb.identifier)
    d1_gmn.app.views.assert_db.is_valid_pid_for_create(pid)
    d1_gmn.app.local_replica.add_to_replication_queue(
        request.POST['sourceNode'], sysmeta_pyxb
    )
    return d1_gmn.app.views.util.http_response_with_boolean_true_type()

Response: MNReplication. replicate - Replicates the sourceNode.


Instruction:
Can you write a function in Python 3 where it
registers this field with a specific process.

Response:
def contribute_to_class(self, process, fields, name):
        """Register this field with a specific process.

        :param process: Process descriptor instance
        :param fields: Fields registry to use
        :param name: Field name
        """
        self.name = name
        self.process = process
        fields[name] = self


Instruction:
Can you create a Python 3 function that
returns a dictionary that represents this field.

Response:
def to_schema(self):
        """Return field schema for this field."""
        if not self.name or not self.process:
            raise ValueError("field is not registered with process")

        schema = {
            'name': self.name,
            'type': self.get_field_type(),
        }
        if self.required is not None:
            schema['required'] = self.required
        if self.label is not None:
            schema['label'] = self.label
        if self.description is not None:
            schema['description'] = self.description
        if self.default is not None:
            schema['default'] = self.default
        if self.hidden is not None:
            schema['hidden'] = self.hidden
        if self.choices is not None:
            for choice, label in self.choices:
                schema.setdefault('choices', []).append({
                    'label': label,
                    'value': choice,
                })

        return schema


Instruction:
Given the following Python 3 function, write the documentation
def clean(self, value):
        if value is None:
            value = self.default

        try:
            value = self.to_python(value)
            self.validate(value)
        except ValidationError as error:
            raise ValidationError("invalid value for {}: {}".format(
                self.name,
                error.args[0]
            ))
        return value

Response: Run validators and return the clean value.


Instruction:
Given the following Python 3 function, write the documentation
def to_output(self, value):
        return json.loads(resolwe_runtime_utils.save(self.name, str(value)))

Response: Convert value to process output format.


Instruction:
Can you write a function in Python 3 where it
converts value if needed.

Response:
def to_python(self, value):
        """Convert value if needed."""
        if isinstance(value, str):
            return value
        elif isinstance(value, dict):
            try:
                value = value['url']
            except KeyError:
                raise ValidationError("dictionary must contain an 'url' element")

            if not isinstance(value, str):
                raise ValidationError("field's url element must be a string")

            return value
        elif not isinstance(value, None):
            raise ValidationError("field must be a string or a dict")


Instruction:
Can you write a function in Python 3 where it
imports the file into the working directory.

Response:
def import_file(self, imported_format=None, progress_from=0.0, progress_to=None):
        """Import field source file to working directory.

        :param imported_format: Import file format (extracted, compressed or both)
        :param progress_from: Initial progress value
        :param progress_to: Final progress value
        :return: Destination file path (if extracted and compressed, extracted path given)
        """
        if not hasattr(resolwe_runtime_utils, 'import_file'):
            raise RuntimeError('Requires resolwe-runtime-utils >= 2.0.0')

        if imported_format is None:
            imported_format = resolwe_runtime_utils.ImportedFormat.BOTH

        return resolwe_runtime_utils.import_file(
            src=self.file_temp,
            file_name=self.path,
            imported_format=imported_format,
            progress_from=progress_from,
            progress_to=progress_to
        )


Instruction:
Implement a function in Python 3 to
convert value if needed.

Response:
def to_python(self, value):
        """Convert value if needed."""
        if isinstance(value, FileDescriptor):
            return value
        elif isinstance(value, str):
            return FileDescriptor(value)
        elif isinstance(value, dict):
            try:
                path = value['file']
            except KeyError:
                raise ValidationError("dictionary must contain a 'file' element")

            if not isinstance(path, str):
                raise ValidationError("field's file element must be a string")

            size = value.get('size', None)
            if size is not None and not isinstance(size, int):
                raise ValidationError("field's size element must be an integer")

            total_size = value.get('total_size', None)
            if total_size is not None and not isinstance(total_size, int):
                raise ValidationError("field's total_size element must be an integer")

            is_remote = value.get('is_remote', None)
            if is_remote is not None and not isinstance(is_remote, bool):
                raise ValidationError("field's is_remote element must be a boolean")

            file_temp = value.get('file_temp', None)
            if file_temp is not None and not isinstance(file_temp, str):
                raise ValidationError("field's file_temp element must be a string")

            refs = value.get('refs', None)
            if refs is not None and not isinstance(refs, list):
                # TODO: Validate that all refs are strings.
                raise ValidationError("field's refs element must be a list of strings")

            return FileDescriptor(
                path,
                size=size,
                total_size=total_size,
                is_remote=is_remote,
                file_temp=file_temp,
                refs=refs,
            )
        elif not isinstance(value, None):
            raise ValidationError("field must be a FileDescriptor, string or a dict")


Instruction:
Write a Python 3 script to
convert value to process output format.

Response:
def to_output(self, value):
        """Convert value to process output format."""
        return json.loads(resolwe_runtime_utils.save_file(self.name, value.path, *value.refs))


Instruction:
How would you implement a function in Python 3 that
converts value if needed.

Response:
def to_python(self, value):
        """Convert value if needed."""
        if isinstance(value, DirDescriptor):
            return value
        elif isinstance(value, str):
            return DirDescriptor(value)
        elif isinstance(value, dict):
            try:
                path = value['dir']
            except KeyError:
                raise ValidationError("dictionary must contain a 'dir' element")

            if not isinstance(path, str):
                raise ValidationError("field's dir element must be a string")

            size = value.get('size', None)
            if size is not None and not isinstance(size, int):
                raise ValidationError("field's size element must be an integer")

            total_size = value.get('total_size', None)
            if total_size is not None and not isinstance(total_size, int):
                raise ValidationError("field's total_size element must be an integer")

            refs = value.get('refs', None)
            if refs is not None and not isinstance(refs, list):
                # TODO: Validate that all refs are strings.
                raise ValidationError("field's refs element must be a list of strings")

            return DirDescriptor(
                path,
                size=size,
                total_size=total_size,
                refs=refs,
            )
        elif not isinstance(value, None):
            raise ValidationError("field must be a DirDescriptor, string or a dict")


Instruction:
Can you write a function in Python 3 where it
registers this field with a specific process.

Response:
def contribute_to_class(self, process, fields, name):
        """Register this field with a specific process.

        :param process: Process descriptor instance
        :param fields: Fields registry to use
        :param name: Field name
        """
        super().contribute_to_class(process, fields, name)

        self.inner.name = name
        self.inner.process = process


Instruction:
Can you write a function in Python 3 where it
converts value to process output format.

Response:
def to_output(self, value):
        """Convert value to process output format."""
        return {self.name: [self.inner.to_output(v)[self.name] for v in value]}


Instruction:
Given the following Python 3 function, write the documentation
def _get(self, key):
        self._populate_cache()
        if key not in self._cache:
            raise AttributeError("DataField has no member {}".format(key))

        return self._cache[key]

Response: Return given key from cache.


Instruction:
Write a Python 3 script for
converting value if needed.

Response:
def to_python(self, value):
        """Convert value if needed."""
        cache = None
        if value is None:
            return None

        if isinstance(value, DataDescriptor):
            return value
        elif isinstance(value, dict):
            # Allow pre-hydrated data objects.
            cache = value
            try:
                value = cache['__id']
            except KeyError:
                raise ValidationError("dictionary must contain an '__id' element")
        elif not isinstance(value, int):
            raise ValidationError("field must be a DataDescriptor, data object primary key or dict")

        return DataDescriptor(value, self, cache)


Instruction:
How would you explain what the following Python 3 function does
def contribute_to_class(self, process, fields, name):
        # Use order-preserving definition namespace (__dict__) to respect the
        # order of GroupField's fields definition.
        for field_name in self.field_group.__dict__:
            if field_name.startswith('_'):
                continue

            field = getattr(self.field_group, field_name)
            field.contribute_to_class(process, self.fields, field_name)

        super().contribute_to_class(process, fields, name)

Response: Register this field with a specific process.


Instruction:
Create a Python 3 function to
convert value if needed.

Response:
def to_python(self, value):
        """Convert value if needed."""
        if isinstance(value, GroupDescriptor):
            value = value._value  # pylint: disable=protected-access

        result = {}
        for name, field in self.fields.items():
            result[name] = field.to_python(value.get(name, None))

        return GroupDescriptor(result)


Instruction:
How would you explain what the following Python 3 function does
def to_schema(self):
        schema = super().to_schema()
        if self.disabled is not None:
            schema['disabled'] = self.disabled
        if self.collapsed is not None:
            schema['collapsed'] = self.collapsed

        group = []
        for field in self.fields.values():
            group.append(field.to_schema())
        schema['group'] = group

        return schema

Response: Return field schema for this field.


Instruction:
Given the following Python 3 function, write the documentation
def update_constants():
    global MANAGER_CONTROL_CHANNEL, MANAGER_EXECUTOR_CHANNELS  # pylint: disable=global-statement
    global MANAGER_LISTENER_STATS, MANAGER_STATE_PREFIX  # pylint: disable=global-statement
    redis_prefix = getattr(settings, 'FLOW_MANAGER', {}).get('REDIS_PREFIX', '')

    MANAGER_CONTROL_CHANNEL = '{}.control'.format(redis_prefix)
    MANAGER_EXECUTOR_CHANNELS = ManagerChannelPair(
        '{}.result_queue'.format(redis_prefix),
        '{}.result_queue_response'.format(redis_prefix),
    )
    MANAGER_STATE_PREFIX = '{}.state'.format(redis_prefix)
    MANAGER_LISTENER_STATS = '{}.listener_stats'.format(redis_prefix)

Response: Recreate channel name constants with changed settings.


Instruction:
Can you tell what is the following Python 3 function doing
def destroy_channels(self):
        for item_name in dir(self):
            item = getattr(self, item_name)
            if isinstance(item, self.RedisAtomicBase):
                self.redis.delete(item.item_name)

Response: Destroy Redis channels managed by this state instance.


Instruction:
Here you have a function in Python 3, explain what it does
def render_to_image_file(
        self, image_out_path, width_pixels=None, height_pixels=None, dpi=90
    ):
        """Render the SubjectInfo to an image file.

        Args:
            image_out_path : str
                Path to where image image will be written. Valid extensions are
                ``.svg,`` ``.pdf``, and ``.png``.

            width_pixels : int
                Width of image to write.

            height_pixels : int
                Height of image to write, in pixels.

            dpi:
                Dots Per Inch to declare in image file. This does not change the
                resolution of the image but may change the size of the image when
                rendered.

        Returns:
            None

        """
        self._render_type = "file"
        self._tree.render(
            file_name=image_out_path,
            w=width_pixels,
            h=height_pixels,
            dpi=dpi,
            units="px",
            tree_style=self._get_tree_style(),
        )

Response: Render the SubjectInfo to an image file.


Instruction:
Implement a Python 3 function for
browsing and edit the SubjectInfo in a simple Qt5 based UI.

Response:
def browse_in_qt5_ui(self):
        """Browse and edit the SubjectInfo in a simple Qt5 based UI."""
        self._render_type = "browse"
        self._tree.show(tree_style=self._get_tree_style())


Instruction:
How would you implement a function in Python 3 that
copies SubjectInfoTree to a ETE Tree.

Response:
def _gen_etetoolkit_tree(self, node, subject_info_tree):
        """Copy SubjectInfoTree to a ETE Tree."""
        for si_node in subject_info_tree.child_list:
            if si_node.type_str == TYPE_NODE_TAG:
                child = self._add_type_node(node, si_node.label_str)
            elif si_node.type_str == SUBJECT_NODE_TAG:
                child = self._add_subject_node(node, si_node.label_str)
            else:
                raise AssertionError(
                    'Unknown node type. type_str="{}"'.format(si_node.type_str)
                )
            self._gen_etetoolkit_tree(child, si_node)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _add_type_node(self, node, label):
        child = node.add_child(name=label)
        child.add_feature(TYPE_NODE_TAG, True)
        return child

Response: Add a node representing a SubjectInfo type.


Instruction:
Can you write a function in Python 3 where it
adds a node containing a subject string.

Response:
def _add_subject_node(self, node, subj_str):
        """Add a node containing a subject string."""
        child = node.add_child(name=subj_str)
        child.add_feature(SUBJECT_NODE_TAG, True)
        return child


Instruction:
Can you create a Python 3 function that
returns the path from the root to node as a list of node names.

Response:
def _get_node_path(self, node):
        """Return the path from the root to ``node`` as a list of node names."""
        path = []
        while node.up:
            path.append(node.name)
            node = node.up
        return list(reversed(path))


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _layout(self, node):

        def set_edge_style():
            """Set the style for edges and make the node invisible."""
            node_style = ete3.NodeStyle()
            node_style["vt_line_color"] = EDGE_COLOR
            node_style["hz_line_color"] = EDGE_COLOR
            node_style["vt_line_width"] = EDGE_WIDTH
            node_style["hz_line_width"] = EDGE_WIDTH
            node_style["size"] = 0
            node.set_style(node_style)

        def style_subject_node(color="Black"):
            """Specify the appearance of Subject nodes."""
            face = ete3.TextFace(node.name, fsize=SUBJECT_NODE_FONT_SIZE, fgcolor=color)
            set_face_margin(face)
            node.add_face(face, column=0, position="branch-right")

        def style_type_node(color="Black"):
            """Specify the appearance of Type nodes."""
            face = ete3.CircleFace(
                radius=TYPE_NODE_RADIUS,
                color=TYPE_NODE_COLOR_DICT.get(node.name, "White"),
                style="circle",
                label={
                    "text": node.name,
                    "color": color,
                    "fontsize": (
                        TYPE_NODE_FONT_SIZE_FILE
                        if self._render_type == "file"
                        else TYPE_NODE_FONT_SIZE_BROWSE
                    ),
                },
            )
            set_face_margin(face)
            node.add_face(face, column=0, position="branch-right")

        def set_face_margin(face):
            """Add margins to Face object.

            - Add space between inner_border and border on TextFace.
            - Add space outside bounding area of CircleFace.

            """
            face.margin_left = 5
            face.margin_right = 5
            # face.margin_top = 5
            # face.margin_bottom = 5

        set_edge_style()

        if hasattr(node, SUBJECT_NODE_TAG):
            style_subject_node()
        elif hasattr(node, TYPE_NODE_TAG):
            style_type_node()
        else:
            raise AssertionError("Unknown node type")

Response: This method renders the node in the ETE file.


Instruction:
Explain what the following Python 3 code does
def extend_settings(self, data_id, files, secrets):
        process = Data.objects.get(pk=data_id).process
        if process.requirements.get('resources', {}).get('secrets', False):
            raise PermissionDenied(
                "Process which requires access to secrets cannot be run using the local executor"
            )

        return super().extend_settings(data_id, files, secrets)

Response: Prevent processes requiring access to secrets from being run.


Instruction:
Can you tell what is the following Python 3 function doing
def get_finder(import_path):
    finder_class = import_string(import_path)
    if not issubclass(finder_class, BaseProcessesFinder):
        raise ImproperlyConfigured(
            'Finder "{}" is not a subclass of "{}"'.format(finder_class, BaseProcessesFinder))
    return finder_class()

Response: Get a process finder.


Instruction:
How would you code a function in Python 3 to
return a list of sub - directories.

Response:
def _find_folders(self, folder_name):
        """Return a list of sub-directories."""
        found_folders = []
        for app_config in apps.get_app_configs():
            folder_path = os.path.join(app_config.path, folder_name)
            if os.path.isdir(folder_path):
                found_folders.append(folder_path)
        return found_folders


Instruction:
Here you have a function in Python 3, explain what it does
def sanity(request, sysmeta_pyxb):
    _does_not_contain_replica_sections(sysmeta_pyxb)
    _is_not_archived(sysmeta_pyxb)
    _obsoleted_by_not_specified(sysmeta_pyxb)
    if 'HTTP_VENDOR_GMN_REMOTE_URL' in request.META:
        return
    _has_correct_file_size(request, sysmeta_pyxb)
    _is_supported_checksum_algorithm(sysmeta_pyxb)
    _is_correct_checksum(request, sysmeta_pyxb)

Response:
Check that sysmeta_pyxb is suitable for creating a new object and matches the
    uploaded sciobj bytes.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def is_valid_sid_for_new_standalone(sysmeta_pyxb):
    sid = d1_common.xml.get_opt_val(sysmeta_pyxb, 'seriesId')
    if not d1_gmn.app.did.is_valid_sid_for_new_standalone(sid):
        raise d1_common.types.exceptions.IdentifierNotUnique(
            0,
            'Identifier is already in use as {}. did="{}"'.format(
                d1_gmn.app.did.classify_identifier(sid), sid
            ),
            identifier=sid,
        )

Response:
Assert that any SID in sysmeta_pyxb can be assigned to a new standalone
    object.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def is_valid_sid_for_chain(pid, sid):
    if not d1_gmn.app.did.is_valid_sid_for_chain(pid, sid):
        existing_sid = d1_gmn.app.revision.get_sid_by_pid(pid)
        raise d1_common.types.exceptions.IdentifierNotUnique(
            0,
            'A different SID is already assigned to the revision chain to which '
            'the object being created or updated belongs. A SID cannot be changed '
            'once it has been assigned to a chain. '
            'existing_sid="{}", new_sid="{}", pid="{}"'.format(existing_sid, sid, pid),
        )

Response: Assert that a SID can be assigned to the single object or to the chain.


Instruction:
Here you have a function in Python 3, explain what it does
def _does_not_contain_replica_sections(sysmeta_pyxb):
    if len(getattr(sysmeta_pyxb, 'replica', [])):
        raise d1_common.types.exceptions.InvalidSystemMetadata(
            0,
            'A replica section was included. A new object object created via '
            'create() or update() cannot already have replicas. pid="{}"'.format(
                d1_common.xml.get_req_val(sysmeta_pyxb.identifier)
            ),
            identifier=d1_common.xml.get_req_val(sysmeta_pyxb.identifier),
        )

Response: Assert that sysmeta_pyxb does not contain any replica information.


Instruction:
Given the following Python 3 function, write the documentation
def _is_not_archived(sysmeta_pyxb):
    if _is_archived(sysmeta_pyxb):
        raise d1_common.types.exceptions.InvalidSystemMetadata(
            0,
            'Archived flag was set. A new object created via create() or update() '
            'cannot already be archived. pid="{}"'.format(
                d1_common.xml.get_req_val(sysmeta_pyxb.identifier)
            ),
            identifier=d1_common.xml.get_req_val(sysmeta_pyxb.identifier),
        )

Response: Assert that sysmeta_pyxb does not have the archived flag set.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def get_d1_env_by_base_url(cn_base_url):
    for k, v in D1_ENV_DICT:
        if v['base_url'].startswith(cn_base_url):
            return D1_ENV_DICT[k]

Response:
Given the BaseURL for a CN return the DataONE environment dict for the CN s
    environemnt.


Instruction:
Can you tell what is the following Python 3 function doing
def match_all(d_SMEFT, parameters=None):
    p = default_parameters.copy()
    if parameters is not None:
        # if parameters are passed in, overwrite the default values
        p.update(parameters)
    C = wilson.util.smeftutil.wcxf2arrays_symmetrized(d_SMEFT)
    C['vT'] = 246.22
    C_WET = match_all_array(C, p)
    C_WET = wilson.translate.wet.rotate_down(C_WET, p)
    C_WET = wetutil.unscale_dict_wet(C_WET)
    d_WET = wilson.util.smeftutil.arrays2wcxf(C_WET)
    basis = wcxf.Basis['WET', 'JMS']
    keys = set(d_WET.keys()) & set(basis.all_wcs)
    d_WET = {k: d_WET[k] for k in keys}
    return d_WET

Response: Match the SMEFT Warsaw basis onto the WET JMS basis.


Instruction:
Implement a function in Python 3 to
return a list of HTTP responses for the debug mode.

Response:
def _debug_mode_responses(self, request, response):
        """Extra functionality available in debug mode.

        - If pretty printed output was requested, force the content type to text. This
          causes the browser to not try to format the output in any way.
        - If SQL profiling is turned on, return a page with SQL query timing
          information instead of the actual response.

        """
        if django.conf.settings.DEBUG_GMN:
            if 'pretty' in request.GET:
                response['Content-Type'] = d1_common.const.CONTENT_TYPE_TEXT
            if (
                'HTTP_VENDOR_PROFILE_SQL' in request.META
                or django.conf.settings.DEBUG_PROFILE_SQL
            ):
                response_list = []
                for query in django.db.connection.queries:
                    response_list.append('{}\n{}'.format(query['time'], query['sql']))
                return django.http.HttpResponse(
                    '\n\n'.join(response_list), d1_common.const.CONTENT_TYPE_TEXT
                )
        return response


Instruction:
Explain what the following Python 3 code does
def queryResponse(
        self, queryEngine, query_str, vendorSpecific=None, do_post=False, **kwargs
    ):
        """CNRead.query(session, queryEngine, query) → OctetStream
        https://releases.dataone.org/online/api-
        documentation-v2.0.1/apis/CN_APIs.html#CNRead.query MNQuery.query(session,
        queryEngine, query) → OctetStream http://jenkins.

        -1.dataone.org/jenkins/job/API%20Documentation%20-%20trunk/ws/api-
        documentation/build/html/apis/MN_APIs.html#MNQuery.query.

        Args:
          queryEngine:
          query_str:
          vendorSpecific:
          do_post:
          **kwargs:

        Returns:

        """
        self._log.debug(
            'Solr query: {}'.format(
                ', '.join(['{}={}'.format(k, v) for (k, v) in list(locals().items())])
            )
        )
        return (self.POST if do_post else self.GET)(
            ['query', queryEngine, query_str], headers=vendorSpecific, **kwargs
        )

Response: Query the ISO for the specified resource set.


Instruction:
How would you code a function in Python 3 to
see Also : queryResponse

Response:
def query(
        self, queryEngine, query_str, vendorSpecific=None, do_post=False, **kwargs
    ):
        """See Also: queryResponse()

        Args:
          queryEngine:
          query_str:
          vendorSpecific:
          do_post:
          **kwargs:

        Returns:

        """
        response = self.queryResponse(
            queryEngine, query_str, vendorSpecific, do_post, **kwargs
        )
        if self._content_type_is_json(response):
            return self._read_json_response(response)
        else:
            return self._read_stream_response(response)


Instruction:
Given the following Python 3 function, write the documentation
def getQueryEngineDescriptionResponse(self, queryEngine, vendorSpecific=None, **kwargs):
        return self.GET(['query', queryEngine], query=kwargs, headers=vendorSpecific)

Response:
CNRead.getQueryEngineDescription(session, queryEngine) →
        QueryEngineDescription https://releases.dataone.org/online/api-
        documentation-v2.0.1/apis/CN_APIs.html#CNRead.getQueryEngineDescription
        MNQuery.getQueryEngineDescription(session, queryEngine) → QueryEngineDescription
        http://jenkins-1.dataone.org/jenkins/job/API%20D ocumentation%20-%20trunk/ws.

        /api-documentation/build/html/apis/MN_APIs.h
        tml#MNQuery.getQueryEngineDescription.

        Args:
          queryEngine:
          **kwargs:

        Returns:


Instruction:
Write a Python 3 function for
seeing Also : getQueryEngineDescriptionResponse

Response:
def getQueryEngineDescription(self, queryEngine, **kwargs):
        """See Also: getQueryEngineDescriptionResponse()

        Args:
          queryEngine:
          **kwargs:

        Returns:

        """
        response = self.getQueryEngineDescriptionResponse(queryEngine, **kwargs)
        return self._read_dataone_type_response(response, 'QueryEngineDescription')


Instruction:
Create a Python 3 function for
deriving cache key for given object.

Response:
def _get_cache_key(self, obj):
        """Derive cache key for given object."""
        if obj is not None:
            # Make sure that key is REALLY unique.
            return '{}-{}'.format(id(self), obj.pk)

        return "{}-None".format(id(self))


Instruction:
Can you generate a brief explanation for the following Python 3 code
def set(self, obj, build_kwargs):
        if build_kwargs is None:
            build_kwargs = {}

        cached = {}
        if 'queryset' in build_kwargs:
            cached = {
                'model': build_kwargs['queryset'].model,
                'pks': list(build_kwargs['queryset'].values_list('pk', flat=True)),
            }

        elif 'obj' in build_kwargs:
            cached = {
                'obj': build_kwargs['obj'],
            }

        if not hasattr(self._thread_local, 'cache'):
            self._thread_local.cache = {}
        self._thread_local.cache[self._get_cache_key(obj)] = cached

Response: Set the cached value for the object.


Instruction:
Can you implement a function in Python 3 that
gets cached value and clean cache.

Response:
def take(self, obj):
        """Get cached value and clean cache."""
        cached = self._thread_local.cache[self._get_cache_key(obj)]
        build_kwargs = {}

        if 'model' in cached and 'pks' in cached:
            build_kwargs['queryset'] = cached['model'].objects.filter(pk__in=cached['pks'])

        elif 'obj' in cached:
            if cached['obj'].__class__.objects.filter(pk=cached['obj'].pk).exists():
                build_kwargs['obj'] = cached['obj']
            else:
                # Object was deleted in the meantime.
                build_kwargs['queryset'] = cached['obj'].__class__.objects.none()

        self._clean_cache(obj)

        return build_kwargs


Instruction:
Here you have a function in Python 3, explain what it does
def connect(self, signal, **kwargs):
        signal.connect(self, **kwargs)
        self.connections.append((signal, kwargs))

Response: Connect a specific signal type to this receiver.


Instruction:
Write a Python 3 function for
disconnecting all connected signal types from this receiver.

Response:
def disconnect(self):
        """Disconnect all connected signal types from this receiver."""
        for signal, kwargs in self.connections:
            signal.disconnect(self, **kwargs)


Instruction:
Implement a Python 3 function for
connecting signals needed for dependency updates.

Response:
def connect(self, index):
        """Connect signals needed for dependency updates.

        Pre- and post-delete signals have to be handled separately, as:

          * in the pre-delete signal we have the information which
            objects to rebuild, but affected relations are still
            presented, so rebuild would reflect in the wrong (outdated)
            indices
          * in the post-delete signal indices can be rebuild corectly,
            but there is no information which objects to rebuild, as
            affected relations were already deleted

        To bypass this, list of objects should be stored in the
        pre-delete signal indexing should be triggered in the
        post-delete signal.
        """
        self.index = index

        signal = ElasticSignal(self, 'process', pass_kwargs=True)
        signal.connect(post_save, sender=self.model)
        signal.connect(pre_delete, sender=self.model)

        pre_delete_signal = ElasticSignal(self, 'process_predelete', pass_kwargs=True)
        pre_delete_signal.connect(pre_delete, sender=self.model)

        post_delete_signal = ElasticSignal(self, 'process_delete', pass_kwargs=True)
        post_delete_signal.connect(post_delete, sender=self.model)

        return [signal, pre_delete_signal, post_delete_signal]


Instruction:
Explain what the following Python 3 code does
def connect(self, index):
        # Determine which model is the target model as either side of the relation
        # may be passed as `field`.
        if index.object_type == self.field.rel.model:
            self.model = self.field.rel.related_model
            self.accessor = self.field.rel.field.attname
        else:
            self.model = self.field.rel.model
            if self.field.rel.symmetrical:
                # Symmetrical m2m relation on self has no reverse accessor.
                raise NotImplementedError(
                    'Dependencies on symmetrical M2M relations are not supported due '
                    'to strange handling of the m2m_changed signal which only makes '
                    'half of the relation visible during signal execution. For now you '
                    'need to use symmetrical=False on the M2M field definition.'
                )
            else:
                self.accessor = self.field.rel.get_accessor_name()

        # Connect signals.
        signals = super().connect(index)

        m2m_signal = ElasticSignal(self, 'process_m2m', pass_kwargs=True)
        m2m_signal.connect(m2m_changed, sender=self.field.through)
        signals.append(m2m_signal)

        # If the relation has a custom through model, we need to subscribe to it.
        if not self.field.rel.through._meta.auto_created:  # pylint: disable=protected-access
            signal = ElasticSignal(self, 'process_m2m_through_save', pass_kwargs=True)
            signal.connect(post_save, sender=self.field.rel.through)
            signals.append(signal)

            signal = ElasticSignal(self, 'process_m2m_through_pre_delete', pass_kwargs=True)
            signal.connect(pre_delete, sender=self.field.rel.through)
            signals.append(signal)

            signal = ElasticSignal(self, 'process_m2m_through_post_delete', pass_kwargs=True)
            signal.connect(post_delete, sender=self.field.rel.through)
            signals.append(signal)

        return signals

Response: Connect signals needed for dependency updates.


Instruction:
Can you create a Python 3 function that
determines if dependent object should be processed.

Response:
def _filter(self, objects, **kwargs):
        """Determine if dependent object should be processed."""
        for obj in objects:
            if self.filter(obj, **kwargs) is False:
                return False

        return True


Instruction:
Make a summary of the following Python 3 code
def _get_build_kwargs(self, obj, pk_set=None, action=None, update_fields=None, reverse=None, **kwargs):
        if action is None:
            # Check filter before rebuilding index.
            if not self._filter([obj], update_fields=update_fields):
                return

            queryset = getattr(obj, self.accessor).all()

            # Special handling for relations to self.
            if self.field.rel.model == self.field.rel.related_model:
                queryset = queryset.union(getattr(obj, self.field.rel.get_accessor_name()).all())

            return {'queryset': queryset}
        else:
            # Update to relation itself, only update the object in question.
            if self.field.rel.model == self.field.rel.related_model:
                # Special case, self-reference, update both ends of the relation.
                pks = set()
                if self._filter(self.model.objects.filter(pk__in=pk_set)):
                    pks.add(obj.pk)

                if self._filter(self.model.objects.filter(pk__in=[obj.pk])):
                    pks.update(pk_set)

                return {'queryset': self.index.object_type.objects.filter(pk__in=pks)}
            elif isinstance(obj, self.model):
                # Need to switch the role of object and pk_set.
                result = {'queryset': self.index.object_type.objects.filter(pk__in=pk_set)}
                pk_set = {obj.pk}
            else:
                result = {'obj': obj}

            if action != 'post_clear':
                # Check filter before rebuilding index.
                if not self._filter(self.model.objects.filter(pk__in=pk_set)):
                    return

            return result

Response: Prepare arguments for rebuilding indices.


Instruction:
How would you code a function in Python 3 to
render the queryset of influenced objects and cache it.

Response:
def process_predelete(self, obj, pk_set=None, action=None, update_fields=None, **kwargs):
        """Render the queryset of influenced objects and cache it."""
        build_kwargs = self._get_build_kwargs(obj, pk_set, action, update_fields, **kwargs)
        self.delete_cache.set(obj, build_kwargs)


Instruction:
Write a Python 3 function that can
recreate queryset from the index and rebuild the index.

Response:
def process_delete(self, obj, pk_set=None, action=None, update_fields=None, **kwargs):
        """Recreate queryset from the index and rebuild the index."""
        build_kwargs = self.delete_cache.take(obj)

        if build_kwargs:
            self.index.build(**build_kwargs)


Instruction:
Create a Python 3 function for
processing signals from dependencies. m2m add signal is processed in two parts.

Response:
def process_m2m(self, obj, pk_set=None, action=None, update_fields=None, cache_key=None, **kwargs):
        """Process signals from dependencies.

        Remove signal is processed in two parts. For details see:
        :func:`~Dependency.connect`
        """
        if action not in (None, 'post_add', 'pre_remove', 'post_remove', 'post_clear'):
            return

        if action == 'post_remove':
            build_kwargs = self.remove_cache.take(cache_key)
        else:
            build_kwargs = self._get_build_kwargs(obj, pk_set, action, update_fields, **kwargs)

        if action == 'pre_remove':
            self.remove_cache.set(cache_key, build_kwargs)
            return

        if build_kwargs:
            self.index.build(**build_kwargs)


Instruction:
Write a Python 3 function for
processing custom M2M through model actions.

Response:
def _process_m2m_through(self, obj, action):
        """Process custom M2M through model actions."""
        source = getattr(obj, self.field.rel.field.m2m_field_name())
        target = getattr(obj, self.field.rel.field.m2m_reverse_field_name())

        pk_set = set()
        if target:
            pk_set.add(target.pk)

        self.process_m2m(source, pk_set, action=action, reverse=False, cache_key=obj)


Instruction:
Can you write a function in Python 3 where it
processes M2M through model post save for custom through model.

Response:
def process_m2m_through_save(self, obj, created=False, **kwargs):
        """Process M2M post save for custom through model."""
        # We are only interested in signals that establish relations.
        if not created:
            return

        self._process_m2m_through(obj, 'post_add')


Instruction:
Can you write a function in Python 3 where it
processes signals from dependencies.

Response:
def process(self, obj, pk_set=None, action=None, update_fields=None, **kwargs):
        """Process signals from dependencies."""
        build_kwargs = self._get_build_kwargs(obj, pk_set, action, update_fields, **kwargs)

        if build_kwargs:
            self.index.build(**build_kwargs)


Instruction:
Here you have a function in Python 3, explain what it does
def _connect_signal(self, index):
        post_save_signal = ElasticSignal(index, 'build')
        post_save_signal.connect(post_save, sender=index.object_type)
        self.signals.append(post_save_signal)

        post_delete_signal = ElasticSignal(index, 'remove_object')
        post_delete_signal.connect(post_delete, sender=index.object_type)
        self.signals.append(post_delete_signal)

        # Connect signals for all dependencies.
        for dependency in index.get_dependencies():
            # Automatically convert m2m fields to dependencies.
            if isinstance(dependency, (models.ManyToManyField, ManyToManyDescriptor)):
                dependency = ManyToManyDependency(dependency)
            elif not isinstance(dependency, Dependency):
                raise TypeError("Unsupported dependency type: {}".format(repr(dependency)))

            signal = dependency.connect(index)
            self.signals.extend(signal)

Response: Connect signals for building indexes.


Instruction:
How would you explain what the following Python 3 function does
def register_signals(self):
        for index in self.indexes:
            if index.object_type:
                self._connect_signal(index)

Response: Register signals for all indexes.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def discover_indexes(self):
        self.indexes = []

        for app_config in apps.get_app_configs():
            indexes_path = '{}.elastic_indexes'.format(app_config.name)
            try:
                indexes_module = import_module(indexes_path)

                for attr_name in dir(indexes_module):
                    attr = getattr(indexes_module, attr_name)
                    if inspect.isclass(attr) and issubclass(attr, BaseIndex) and attr is not BaseIndex:
                        # Make sure that parallel tests have different indices.
                        if is_testing():
                            index = attr.document_class._index._name  # pylint: disable=protected-access
                            testing_postfix = '_test_{}_{}'.format(TESTING_UUID, os.getpid())

                            if not index.endswith(testing_postfix):
                                # Replace current postfix with the new one.
                                if attr.testing_postfix:
                                    index = index[:-len(attr.testing_postfix)]
                                index = index + testing_postfix
                                attr.testing_postfix = testing_postfix

                            attr.document_class._index._name = index  # pylint: disable=protected-access

                        index = attr()

                        # Apply any extensions defined for the given index. Currently index extensions are
                        # limited to extending "mappings".
                        for extension in composer.get_extensions(attr):
                            mapping = getattr(extension, 'mapping', {})
                            index.mapping.update(mapping)

                        self.indexes.append(index)
            except ImportError as ex:
                if not re.match('No module named .*elastic_indexes.*', str(ex)):
                    raise

Response: Find all indexes and add them to the _index_builders attribute.


Instruction:
Explain what the following Python 3 code does
def build(self, obj=None, queryset=None, push=True):
        for index in self.indexes:
            index.build(obj, queryset, push)

Response: Trigger building of the indexes.


Instruction:
Write a Python 3 script for
pushing built documents to ElasticSearch.

Response:
def push(self, index=None):
        """Push built documents to ElasticSearch.

        If ``index`` is specified, only that index will be pushed.
        """
        for ind in self.indexes:
            if index and not isinstance(ind, index):
                continue
            ind.push()


Instruction:
Implement a function in Python 3 to
delete all entries from ElasticSearch.

Response:
def delete(self, skip_mapping=False):
        """Delete all entries from ElasticSearch."""
        for index in self.indexes:
            index.destroy()
            if not skip_mapping:
                index.create_mapping()


Instruction:
Implement a Python 3 function for
deleting all indexes from Elasticsearch and index builder.

Response:
def destroy(self):
        """Delete all indexes from Elasticsearch and index builder."""
        self.unregister_signals()
        for index in self.indexes:
            index.destroy()
        self.indexes = []


Instruction:
Make a summary of the following Python 3 code
def _set_initial(self, C_in, scale_in):
        self.C_in = C_in
        self.scale_in = scale_in

Response:
Set the initial values for parameters and Wilson coefficients at
        the scale scale_in.


Instruction:
Can you tell what is the following Python 3 function doing
def _set_initial_wcxf(self, wc, get_smpar=True):
        if wc.eft != 'SMEFT':
            raise ValueError("Wilson coefficients use wrong EFT.")
        if wc.basis != 'Warsaw':
            raise ValueError("Wilson coefficients use wrong basis.")
        self.scale_in = wc.scale
        C = wilson.util.smeftutil.wcxf2arrays_symmetrized(wc.dict)
        # fill in zeros for missing WCs
        for k, s in smeftutil.C_keys_shape.items():
            if k not in C and k not in smeftutil.SM_keys:
                if s == 1:
                    C[k] = 0
                else:
                    C[k] = np.zeros(s)
        if self.C_in is None:
            self.C_in = C
        else:
            self.C_in.update(C)
        if get_smpar:
            self.C_in.update(self._get_sm_scale_in())

Response: Load the initial values for the Wilson coefficients from a Wilson coefficients object.


Instruction:
Explain what the following Python 3 code does
def _to_wcxf(self, C_out, scale_out):
        C = self._rotate_defaultbasis(C_out)
        d = wilson.util.smeftutil.arrays2wcxf_nonred(C)
        basis = wcxf.Basis['SMEFT', 'Warsaw']
        all_wcs = set(basis.all_wcs)  # to speed up lookup
        d = {k: v for k, v in d.items() if k in all_wcs and v != 0}
        d = wcxf.WC.dict2values(d)
        wc = wcxf.WC('SMEFT', 'Warsaw', scale_out, d)
        return wc

Response: Return the Wilson coefficients C_out as a wcxf. WC instance.


Instruction:
Make a summary of the following Python 3 code
def _rgevolve(self, scale_out, **kwargs):
        self._check_initial()
        return rge.smeft_evolve(C_in=self.C_in,
                            scale_in=self.scale_in,
                            scale_out=scale_out,
                            **kwargs)

Response: Solve the SMEFT RGEs from the initial scale to scale_out.


Instruction:
Create a Python 3 function for
computing the leading logarithmic approximation to the solution of the SMEFT RGEs from the initial scale to scale_out.

Response:
def _rgevolve_leadinglog(self, scale_out):
        """Compute the leading logarithmic approximation to the solution
        of the SMEFT RGEs from the initial scale to `scale_out`.
        Returns a dictionary with parameters and Wilson coefficients.
        Much faster but less precise that `rgevolve`.
        """
        self._check_initial()
        return rge.smeft_evolve_leadinglog(C_in=self.C_in,
                            scale_in=self.scale_in,
                            scale_out=scale_out)


Instruction:
How would you explain what the following Python 3 function does
def _run_sm_scale_in(self, C_out, scale_sm=91.1876):
        # initialize an empty SMEFT instance
        smeft_sm = SMEFT(wc=None)
        C_in_sm = smeftutil.C_array2dict(np.zeros(9999))
        # set the SM parameters to the values obtained from smpar.smeftpar
        C_SM = smpar.smeftpar(scale_sm, C_out, basis='Warsaw')
        SM_keys = set(smeftutil.SM_keys)  # to speed up lookup
        C_SM = {k: v for k, v in C_SM.items() if k in SM_keys}
        # set the Wilson coefficients at the EW scale to C_out
        C_in_sm.update(C_out)
        C_in_sm.update(C_SM)
        smeft_sm._set_initial(C_in_sm, scale_sm)
        # run up (with 1% relative precision, ignore running of Wilson coefficients)
        C_SM_high = smeft_sm._rgevolve(self.scale_in, newphys=False, rtol=0.001, atol=1)
        C_SM_high = self._rotate_defaultbasis(C_SM_high)
        return {k: v for k, v in C_SM_high.items() if k in SM_keys}

Response:
Get the SM parameters at the EW scale using an estimate C_out and run them to the
        input scale.


Instruction:
Create a Python 3 function for
computing the SM - bar parameters at the electroweak scale.

Response:
def get_smpar(self, accuracy='integrate', scale_sm=91.1876):
        """Compute the SM MS-bar parameters at the electroweak scale.

        This method can be used to validate the accuracy of the iterative
        extraction of SM parameters. If successful, the values returned by this
        method should agree with the values in the dictionary
        `wilson.run.smeft.smpar.p`."""
        if accuracy == 'integrate':
            C_out = self._rgevolve(scale_sm)
        elif accuracy == 'leadinglog':
            C_out = self._rgevolve_leadinglog(scale_sm)
        else:
            raise ValueError("'{}' is not a valid value of 'accuracy' (must be either 'integrate' or 'leadinglog').".format(accuracy))
        return smpar.smpar(C_out)


Instruction:
Write a Python 3 script for
running the SM up and return the SM parameters at the input scale.

Response:
def _get_sm_scale_in(self, scale_sm=91.1876):
        """Get an estimate of the SM parameters at the input scale by running
        them from the EW scale using constant values for the Wilson coefficients
        (corresponding to their leading log approximated values at the EW
        scale).

        Note that this is not guaranteed to work and will fail if some of the
        Wilson coefficients (the ones affecting the extraction of SM parameters)
        are large."""
        # intialize a copy of ourselves
        _smeft = SMEFT(self.wc, get_smpar=False)
        # Step 1: run the SM up, using the WCs at scale_input as (constant) estimate
        _smeft.C_in.update(self._run_sm_scale_in(self.C_in, scale_sm=scale_sm))
        # Step 2: run the WCs down in LL approximation
        C_out = _smeft._rgevolve_leadinglog(scale_sm)
        # Step 3: run the SM up again, this time using the WCs at scale_sm as (constant) estimate
        return self._run_sm_scale_in(C_out, scale_sm=scale_sm)


Instruction:
Make a summary of the following Python 3 code
def run(self, scale, accuracy='integrate', **kwargs):
        if accuracy == 'integrate':
            C_out = self._rgevolve(scale, **kwargs)
        elif accuracy == 'leadinglog':
            C_out = self._rgevolve_leadinglog(scale)
        else:
            raise ValueError("'{}' is not a valid value of 'accuracy' (must be either 'integrate' or 'leadinglog').".format(accuracy))
        return self._to_wcxf(C_out, scale)

Response: Returns the Wilson coefficients evolved to the specified scale.


Instruction:
Can you write a function in Python 3 where it
returns a continuous solution to the RGE as RGsolution instance.

Response:
def run_continuous(self, scale):
        """Return a continuous solution to the RGE as `RGsolution` instance."""
        if scale == self.scale_in:
            raise ValueError("The scale must be different from the input scale")
        elif scale < self.scale_in:
            scale_min = scale
            scale_max = self.scale_in
        elif scale > self.scale_in:
            scale_max = scale
            scale_min = self.scale_in
        fun = rge.smeft_evolve_continuous(C_in=self.C_in,
                                          scale_in=self.scale_in,
                                          scale_out=scale)
        return wilson.classes.RGsolution(fun, scale_min, scale_max)


Instruction:
How would you implement a function in Python 3 that
returns SQL filter for filtering by fields in unique_with attribute.

Response:
def _get_unique_constraints(self, instance):
        """Return SQL filter for filtering by fields in ``unique_with`` attribute.

        Filter is returned as tuple of two elements where first one is
        placeholder which is safe to insert into SQL query and second
        one may include potentially dangerous values and must be passed
        to SQL query in ``params`` attribute to make sure it is properly
        escaped.
        """
        constraints_expression = []
        constraints_values = {}
        for field_name in self.unique_with:
            if constants.LOOKUP_SEP in field_name:
                raise NotImplementedError(
                    '`unique_with` constraint does not support lookups by related models.'
                )

            field = instance._meta.get_field(field_name)  # pylint: disable=protected-access
            field_value = getattr(instance, field_name)

            # Convert value to the database representation.
            field_db_value = field.get_prep_value(field_value)

            constraint_key = 'unique_' + field_name
            constraints_expression.append("{} = %({})s".format(
                connection.ops.quote_name(field.column),
                constraint_key
            ))
            constraints_values[constraint_key] = field_db_value

        if not constraints_expression:
            return '', []

        constraints_expression = 'AND ' + ' AND '.join(constraints_expression)
        return constraints_expression, constraints_values


Instruction:
Here you have a function in Python 3, explain what it does
def _get_populate_from_value(self, instance):
        if hasattr(self.populate_from, '__call__'):
            # ResolweSlugField(populate_from=lambda instance: ...)
            return self.populate_from(instance)
        else:
            # ResolweSlugField(populate_from='foo')
            attr = getattr(instance, self.populate_from)
            return attr() if callable(attr) else attr

Response: Get the value from populate_from attribute.


Instruction:
Can you create a Python 3 function that
ensures slug uniqunes before save.

Response:
def pre_save(self, instance, add):
        """Ensure slug uniqunes before save."""
        slug = self.value_from_object(instance)

        # We don't want to change slug defined by user.
        predefined_slug = bool(slug)

        if not slug and self.populate_from:
            slug = self._get_populate_from_value(instance)

        if slug:
            slug = slugify(slug)

        if not slug:
            slug = None

            if not self.blank:
                slug = instance._meta.model_name  # pylint: disable=protected-access
            elif not self.null:
                slug = ''

        if slug:
            # Make sure that auto generated slug with added sequence
            # won't excede maximal length.
            # Validation of predefined slugs is handled by Django.
            if not predefined_slug:
                slug = slug[:(self.max_length - MAX_SLUG_SEQUENCE_DIGITS - 1)]

            constraints_placeholder, constraints_values = self._get_unique_constraints(instance)

            instance_pk_name = instance._meta.pk.name  # pylint: disable=protected-access

            # Safe values - make sure that there is no chance of SQL injection.
            query_params = {
                'constraints_placeholder': constraints_placeholder,
                'slug_column': connection.ops.quote_name(self.column),
                'slug_len': len(slug),
                'table_name': connection.ops.quote_name(self.model._meta.db_table),  # pylint: disable=protected-access
                'pk_neq_placeholder': 'AND {} != %(instance_pk)s'.format(instance_pk_name) if instance.pk else ''
            }

            # SQL injection unsafe values - will be escaped.
            # Keys prefixed with `unique_` are reserved for `constraints_values` dict.
            query_escape_params = {
                'slug': slug,
                'slug_regex': '^{}(-[0-9]*)?$'.format(slug),
            }
            query_escape_params.update(constraints_values)
            if instance.pk:
                query_escape_params['instance_pk'] = instance.pk

            with connection.cursor() as cursor:
                # TODO: Slowest part of this query is `MAX` function. It can
                #       be optimized by indexing slug column by slug sequence.
                #       https://www.postgresql.org/docs/9.4/static/indexes-expressional.html
                cursor.execute(
                    """
                    SELECT
                        CASE
                            WHEN (
                                EXISTS(
                                    SELECT 1 FROM {table_name} WHERE (
                                        {slug_column} = %(slug)s
                                        {pk_neq_placeholder}
                                        {constraints_placeholder}
                                    )
                                )
                            ) THEN MAX(slug_sequence) + 1
                            ELSE NULL
                        END
                    FROM (
                        SELECT COALESCE(
                            NULLIF(
                                RIGHT({slug_column}, -{slug_len}-1),
                                ''
                            ),
                            '1'
                        )::text::integer AS slug_sequence
                        FROM {table_name} WHERE (
                            {slug_column} ~ %(slug_regex)s
                            {pk_neq_placeholder}
                            {constraints_placeholder}
                        )
                    ) AS tmp
                    """.format(**query_params),
                    params=query_escape_params
                )
                result = cursor.fetchone()[0]

            if result is not None:
                if predefined_slug:
                    raise SlugError(
                        "Slug '{}' (version {}) is already taken.".format(slug, instance.version)
                    )

                if len(str(result)) > MAX_SLUG_SEQUENCE_DIGITS:
                    raise SlugError(
                        "Auto-generated slug sequence too long - please choose a different slug."
                    )

                slug = '{}-{}'.format(slug, result)

            # Make the updated slug available as instance attribute.
            setattr(instance, self.name, slug)

        return slug


Instruction:
How would you code a function in Python 3 to
perform process discovery in given path.

Response:
def discover_process(self, path):
        """Perform process discovery in given path.

        This method will be called during process registration and
        should return a list of dictionaries with discovered process
        schemas.
        """
        if not path.lower().endswith(('.yml', '.yaml')):
            return []

        with open(path) as fn:
            schemas = yaml.load(fn, Loader=yaml.FullLoader)
        if not schemas:
            # TODO: Logger.
            # self.stderr.write("Could not read YAML file {}".format(schema_file))
            return []

        process_schemas = []
        for schema in schemas:
            if 'run' not in schema:
                continue

            # NOTE: This currently assumes that 'bash' is the default.
            if schema['run'].get('language', 'bash') != 'workflow':
                continue

            process_schemas.append(schema)

        return process_schemas


Instruction:
Given the following Python 3 function, write the documentation
def _evaluate_expressions(self, expression_engine, step_id, values, context):
        if expression_engine is None:
            return values

        processed = {}
        for name, value in values.items():
            if isinstance(value, str):
                value = value.strip()
                try:
                    expression = expression_engine.get_inline_expression(value)
                    if expression is not None:
                        # Inline expression.
                        value = expression_engine.evaluate_inline(expression, context)
                    else:
                        # Block expression.
                        value = expression_engine.evaluate_block(value, context)
                except EvaluationError as error:
                    raise ExecutionError('Error while evaluating expression for step "{}":\n{}'.format(
                        step_id, error
                    ))
            elif isinstance(value, dict):
                value = self._evaluate_expressions(expression_engine, step_id, value, context)

            processed[name] = value

        return processed

Response: Recursively evaluate expressions in a dictionary of values.


Instruction:
Can you write a function in Python 3 where it
evaluates the code needed to compute a given Data object.

Response:
def evaluate(self, data):
        """Evaluate the code needed to compute a given Data object."""
        expression_engine = data.process.requirements.get('expression-engine', None)
        if expression_engine is not None:
            expression_engine = self.get_expression_engine(expression_engine)

        # Parse steps.
        steps = data.process.run.get('program', None)
        if steps is None:
            return

        if not isinstance(steps, list):
            raise ExecutionError('Workflow program must be a list of steps.')

        # Expression engine evaluation context.
        context = {
            'input': data.input,
            'steps': collections.OrderedDict(),
        }

        for index, step in enumerate(steps):
            try:
                step_id = step['id']
                step_slug = step['run']
            except KeyError as error:
                raise ExecutionError('Incorrect definition of step "{}", missing property "{}".'.format(
                    step.get('id', index), error
                ))

            # Fetch target process.
            process = Process.objects.filter(slug=step_slug).order_by('-version').first()
            if not process:
                raise ExecutionError('Incorrect definition of step "{}", invalid process "{}".'.format(
                    step_id, step_slug
                ))

            # Process all input variables.
            step_input = step.get('input', {})
            if not isinstance(step_input, dict):
                raise ExecutionError('Incorrect definition of step "{}", input must be a dictionary.'.format(
                    step_id
                ))

            data_input = self._evaluate_expressions(expression_engine, step_id, step_input, context)

            # Create the data object.
            data_object = Data.objects.create(
                process=process,
                contributor=data.contributor,
                tags=data.tags,
                input=data_input,
            )
            DataDependency.objects.create(
                parent=data,
                child=data_object,
                kind=DataDependency.KIND_SUBPROCESS,
            )

            # Copy permissions.
            copy_permissions(data, data_object)

            # Copy collections.
            for collection in data.collection_set.all():
                collection.data.add(data_object)

            context['steps'][step_id] = data_object.pk

        # Immediately set our status to done and output all data object identifiers.
        data.output = {
            'steps': list(context['steps'].values()),
        }
        data.status = Data.STATUS_DONE


Instruction:
How would you implement a function in Python 3 that
creates a connection to the Redis server.

Response:
async def init():
    """Create a connection to the Redis server."""
    global redis_conn  # pylint: disable=global-statement,invalid-name
    conn = await aioredis.create_connection(
        'redis://{}:{}'.format(
            SETTINGS.get('FLOW_EXECUTOR', {}).get('REDIS_CONNECTION', {}).get('host', 'localhost'),
            SETTINGS.get('FLOW_EXECUTOR', {}).get('REDIS_CONNECTION', {}).get('port', 56379)
        ),
        db=int(SETTINGS.get('FLOW_EXECUTOR', {}).get('REDIS_CONNECTION', {}).get('db', 1))
    )
    redis_conn = aioredis.Redis(conn)


Instruction:
Can you generate the documentation for the following Python 3 function
async def send_manager_command(cmd, expect_reply=True, extra_fields={}):
    packet = {
        ExecutorProtocol.DATA_ID: DATA['id'],
        ExecutorProtocol.COMMAND: cmd,
    }
    packet.update(extra_fields)

    logger.debug("Sending command to listener: {}".format(json.dumps(packet)))

    # TODO what happens here if the push fails? we don't have any realistic recourse,
    # so just let it explode and stop processing
    queue_channel = EXECUTOR_SETTINGS['REDIS_CHANNEL_PAIR'][0]
    try:
        await redis_conn.rpush(queue_channel, json.dumps(packet))
    except Exception:
        logger.error("Error sending command to manager:\n\n{}".format(traceback.format_exc()))
        raise

    if not expect_reply:
        return

    for _ in range(_REDIS_RETRIES):
        response = await redis_conn.blpop(QUEUE_RESPONSE_CHANNEL, timeout=1)
        if response:
            break
    else:
        # NOTE: If there's still no response after a few seconds, the system is broken
        # enough that it makes sense to give up; we're isolated here, so if the manager
        # doesn't respond, we can't really do much more than just crash
        raise RuntimeError("No response from the manager after {} retries.".format(_REDIS_RETRIES))

    _, item = response
    result = json.loads(item.decode('utf-8'))[ExecutorProtocol.RESULT]
    assert result in [ExecutorProtocol.RESULT_OK, ExecutorProtocol.RESULT_ERROR]

    if result == ExecutorProtocol.RESULT_OK:
        return True

    return False

Response: Send a properly formatted command to the manager.


Instruction:
How would you implement a function in Python 3 that
extracts values from one SciObj.

Response:
def extract_values_query(query, field_list, out_stream=None):
    """Get list of dicts where each dict holds values from one SciObj.

    Args:
        field_list: list of str
            List of field names for which to return values. Must be strings from
            FIELD_NAME_TO_generate_dict.keys().

            If None, return all fields.

        filter_arg_dict: dict
            Dict of arguments to pass to ``ScienceObject.objects.filter()``.

        out_stream: open file-like object
            If provided, the JSON doc is streamed out instead of buffered in memory.

    Returns:
        list of dict: The keys in the returned dict correspond to the field names in
        ``field_list``.

    Raises:
        raise d1_common.types.exceptions.InvalidRequest() if ``field_list`` contains any
        invalid field names. A list of the invalid fields is included in the exception.

    """
    lookup_dict, generate_dict = _split_field_list(field_list)

    query, annotate_key_list = _annotate_query(query, generate_dict)
    # return query, annotate_key_list
    #
    # query, annotate_key_list = _create_query(filter_arg_dict, generate_dict)

    lookup_list = [v["lookup_str"] for k, v in lookup_dict.items()] + annotate_key_list

    if out_stream is None:
        return _create_sciobj_list(query, lookup_list, lookup_dict, generate_dict)
    else:
        return _write_stream(query, lookup_list, lookup_dict, generate_dict, out_stream)


Instruction:
Explain what the following Python 3 code does
def extract_values(field_list=None, filter_arg_dict=None, out_stream=None):
    lookup_dict, generate_dict = _split_field_list(field_list)
    query, annotate_key_list = _create_query(filter_arg_dict, generate_dict)

    lookup_list = [v["lookup_str"] for k, v in lookup_dict.items()] + annotate_key_list

    if out_stream is None:
        return _create_sciobj_list(query, lookup_list, lookup_dict, generate_dict)
    else:
        return _write_stream(query, lookup_list, lookup_dict, generate_dict, out_stream)

Response: Extract values from one SciObj.


Instruction:
Write a Python 3 script to
raise an exception if the given list contains invalid field names.

Response:
def assert_invalid_field_list(field_list):
    """raise d1_common.types.exceptions.InvalidRequest() if ``field_list`` contains any
    invalid field names. A list of the invalid fields is included in the exception.

    - Implicitly called by ``extract_values()``.

    """
    if field_list is not None:
        invalid_field_list = [
            v for v in field_list if v not in get_valid_field_name_list()
        ]
        if invalid_field_list:
            raise d1_common.types.exceptions.InvalidRequest(
                0, "Invalid fields: {}".format(", ".join(invalid_field_list))
            )


Instruction:
Implement a Python 3 function for
adding annotations to the query to retrieve values required by field value generate functions.

Response:
def _annotate_query(query, generate_dict):
    """Add annotations to the query to retrieve values required by field value generate
    functions."""
    annotate_key_list = []
    for field_name, annotate_dict in generate_dict.items():
        for annotate_name, annotate_func in annotate_dict["annotate_dict"].items():
            query = annotate_func(query)
            annotate_key_list.append(annotate_name)
    return query, annotate_key_list


Instruction:
Write a Python 3 script for
creating a dict where the keys are the requested field names from the values returned by Django.

Response:
def _value_list_to_sciobj_dict(
    sciobj_value_list, lookup_list, lookup_dict, generate_dict
):
    """Create a dict where the keys are the requested field names, from the values
    returned by Django."""

    sciobj_dict = {}

    # for sciobj_value, lookup_str in zip(sciobj_value_list, lookup_list):
    lookup_to_value_dict = {k: v for k, v in zip(lookup_list, sciobj_value_list)}

    for field_name, r_dict in lookup_dict.items():
        if r_dict["lookup_str"] in lookup_to_value_dict.keys():
            sciobj_dict[field_name] = lookup_to_value_dict[r_dict["lookup_str"]]

    for field_name, annotate_dict in generate_dict.items():
        for final_name, generate_func in annotate_dict["generate_dict"].items():
            sciobj_dict[field_name] = generate_func(lookup_to_value_dict)

    return sciobj_dict


Instruction:
Make a summary of the following Python 3 code
def _split_field_list(field_list):
    lookup_dict = {}
    generate_dict = {}

    for field_name in field_list or FIELD_NAME_TO_EXTRACT_DICT.keys():
        try:
            extract_dict = FIELD_NAME_TO_EXTRACT_DICT[field_name]
        except KeyError:
            assert_invalid_field_list(field_list)
        else:
            if "lookup_str" in extract_dict:
                lookup_dict[field_name] = extract_dict
            else:
                generate_dict[field_name] = extract_dict

    return lookup_dict, generate_dict

Response:
Split the list of fields for which to extract values into lists by extraction
    methods.


Instruction:
Create a Python 3 function for
returning the PyXB binding to use when handling a request.

Response:
def dataoneTypes(request):
    """Return the PyXB binding to use when handling a request."""
    if is_v1_api(request):
        return d1_common.types.dataoneTypes_v1_1
    elif is_v2_api(request) or is_diag_api(request):
        return d1_common.types.dataoneTypes_v2_0
    else:
        raise d1_common.types.exceptions.ServiceFailure(
            0, 'Unknown version designator in URL. url="{}"'.format(request.path)
        )


Instruction:
Write a Python 3 script for
parsing a ISO 8601 date - time and return a datetime with timezone adjusted to UTC.

Response:
def parse_and_normalize_url_date(date_str):
    """Parse a ISO 8601 date-time with optional timezone.

    - Return as datetime with timezone adjusted to UTC.
    - Return naive date-time set to UTC.

    """
    if date_str is None:
        return None
    try:
        return d1_common.date_time.dt_from_iso8601_str(date_str)
    except d1_common.date_time.iso8601.ParseError as e:
        raise d1_common.types.exceptions.InvalidRequest(
            0,
            'Invalid date format for URL parameter. date="{}" error="{}"'.format(
                date_str, str(e)
            ),
        )


Instruction:
Write a Python 3 script for
retrieving the specified document.

Response:
def get(self, doc_id):
        """Retrieve the specified document."""
        resp_dict = self._get_query(q='id:{}'.format(doc_id))
        if resp_dict['response']['numFound'] > 0:
            return resp_dict['response']['docs'][0]


Instruction:
Write a Python 3 function that can
retrieve a list of identifiers for documents matching the query.

Response:
def get_ids(self, start=0, rows=1000, **query_dict):
        """Retrieve a list of identifiers for documents matching the query."""
        resp_dict = self._get_query(start=start, rows=rows, **query_dict)
        return {
            'matches': resp_dict['response']['numFound'],
            'start': start,
            'ids': [d['id'] for d in resp_dict['response']['docs']],
        }


Instruction:
How would you code a function in Python 3 to
return the number of entries that match query_dict.

Response:
def count(self, **query_dict):
        """Return the number of entries that match query."""
        param_dict = query_dict.copy()
        param_dict['count'] = 0
        resp_dict = self._get_query(**param_dict)
        return resp_dict['response']['numFound']


Instruction:
Can you generate a brief explanation for the following Python 3 code
def get_field_values(self, name, maxvalues=-1, sort=True, **query_dict):
        param_dict = query_dict.copy()
        param_dict.update(
            {
                'rows': '0',
                'facet': 'true',
                'facet.field': name,
                'facet.limit': str(maxvalues),
                'facet.zeros': 'false',
                'facet.sort': str(sort).lower(),
            }
        )
        resp_dict = self._post_query(**param_dict)
        result_dict = resp_dict['facet_counts']['facet_fields']
        result_dict['numFound'] = resp_dict['response']['numFound']
        return result_dict

Response: Retrieve the unique values for a field.