Instruction:
How would you explain what the following Python 3 function does
def create_modules_toc_file(master_package, modules, opts, name='modules'):
    text = format_heading(1, '%s Modules' % opts.header)
    text += '.. toctree::\n'
    text += '   :maxdepth: %s\n\n' % opts.maxdepth

    modules.sort()
    prev_module = ''
    for module in modules:
        # look if the module is a subpackage and, if yes, ignore it
        if module.startswith(prev_module + '.'):
            continue
        prev_module = module
        text += '   %s\n' % module

    write_file(name, text, opts)

Response: Create the modules toc file.


Instruction:
Write a Python 3 script for
looking for every file in the directory tree and create the corresponding ReST files.

Response:
def recurse_tree(path, excludes, opts):
    """
    Look for every file in the directory tree and create the corresponding
    ReST files.
    """
    # use absolute path for root, as relative paths like '../../foo' cause
    # 'if "/." in root ...' to filter out *all* modules otherwise
    path = os.path.abspath(path)
    # check if the base directory is a package and get is name
    if INIT in os.listdir(path):
        package_name = path.split(os.path.sep)[-1]
    else:
        package_name = None

    toc = []
    tree = os.walk(path, False)
    for root, subs, files in tree:
        # keep only the Python script files
        py_files =  sorted([f for f in files if os.path.splitext(f)[1] == '.py'])
        if INIT in py_files:
            py_files.remove(INIT)
            py_files.insert(0, INIT)
        # remove hidden ('.') and private ('_') directories
        subs = sorted([sub for sub in subs if sub[0] not in ['.', '_']])
        # check if there are valid files to process
        # TODO: could add check for windows hidden files
        if "/." in root or "/_" in root \
        or not py_files \
        or is_excluded(root, excludes):
            continue
        if INIT in py_files:
            # we are in package ...
            if (# ... with subpackage(s)
                subs
                or
                # ... with some module(s)
                len(py_files) > 1
                or
                # ... with a not-to-be-skipped INIT file
                not shall_skip(os.path.join(root, INIT))
               ):
                subroot = root[len(path):].lstrip(os.path.sep).replace(os.path.sep, '.')
                create_package_file(root, package_name, subroot, py_files, opts, subs)
                toc.append(makename(package_name, subroot))
        elif root == path:
            # if we are at the root level, we don't require it to be a package
            for py_file in py_files:
                if not shall_skip(os.path.join(path, py_file)):
                    module = os.path.splitext(py_file)[0]
                    create_module_file(package_name, module, opts)
                    toc.append(makename(package_name, module))

    # create the module's index
    if not opts.notoc:
        create_modules_toc_file(package_name, toc, opts)


Instruction:
Can you tell what is the following Python 3 function doing
def normalize_excludes(rootpath, excludes):
    sep = os.path.sep
    f_excludes = []
    for exclude in excludes:
        if not os.path.isabs(exclude) and not exclude.startswith(rootpath):
            exclude = os.path.join(rootpath, exclude)
        if not exclude.endswith(sep):
            exclude += sep
        f_excludes.append(exclude)
    return f_excludes

Response: Normalize the excluded directory list


Instruction:
Here you have a function in Python 3, explain what it does
def is_excluded(root, excludes):
    sep = os.path.sep
    if not root.endswith(sep):
        root += sep
    for exclude in excludes:
        if root.startswith(exclude):
            return True
    return False

Response: Check if the directory is in the exclude list.


Instruction:
Make a summary of the following Python 3 code
def main():
    parser = optparse.OptionParser(usage="""usage: %prog [options] <package path> [exclude paths, ...]

Note: By default this script will not overwrite already created files.""")
    parser.add_option("-n", "--doc-header", action="store", dest="header", help="Documentation Header (default=Project)", default="Project")
    parser.add_option("-d", "--dest-dir", action="store", dest="destdir", help="Output destination directory", default="")
    parser.add_option("-s", "--suffix", action="store", dest="suffix", help="module suffix (default=txt)", default="txt")
    parser.add_option("-m", "--maxdepth", action="store", dest="maxdepth", help="Maximum depth of submodules to show in the TOC (default=4)", type="int", default=4)
    parser.add_option("-r", "--dry-run", action="store_true", dest="dryrun", help="Run the script without creating the files")
    parser.add_option("-f", "--force", action="store_true", dest="force", help="Overwrite all the files")
    parser.add_option("-t", "--no-toc", action="store_true", dest="notoc", help="Don't create the table of content file")
    (opts, args) = parser.parse_args()
    if not args:
        parser.error("package path is required.")
    else:
        rootpath, excludes = args[0], args[1:]
        if os.path.isdir(rootpath):
            # check if the output destination is a valid directory
            if opts.destdir and os.path.isdir(opts.destdir):
                excludes = normalize_excludes(rootpath, excludes)
                recurse_tree(rootpath, excludes, opts)
            else:
                print '%s is not a valid output destination directory.' % opts.destdir
        else:
            print '%s is not a valid directory.' % rootpath

Response: Parse and check the command line arguments and run the main function.


Instruction:
How would you explain what the following Python 3 function does
def path(self):

        if not isinstance(self.ref, str):
            return None

        u = parse_app_url(self.ref)

        if u.inner.proto != 'file':
            return None

        return u.path

Response: Return the path to the file


Instruction:
Given the following Python 3 function, write the documentation
def doc_dir(self):
        from os.path import abspath

        if not self.ref:
            return None

        u = parse_app_url(self.ref)
        return abspath(dirname(u.path))

Response: The absolute directory of the document


Instruction:
Explain what the following Python 3 code does
def new_section(self, name, params=None):
        self.sections[name.lower()] = SectionTerm(None, name, term_args=params, doc=self)

        # Set the default arguments
        s = self.sections[name.lower()]

        if name.lower() in self.decl_sections:
            s.args = self.decl_sections[name.lower()]['args']

        return s

Response: Return a new section


Instruction:
Can you generate a brief explanation for the following Python 3 code
def get_or_new_section(self, name, params=None):
        if name not in self.sections:
            self.sections[name.lower()] = SectionTerm(None, name, term_args=params, doc=self)

        return self.sections[name.lower()]

Response: Create a new section or return an existing one of the same name


Instruction:
How would you code a function in Python 3 to
sort the sections according to the section names in the order list.

Response:
def sort_sections(self, order):
        """
        Sort sections according to the section names in the order list. All remaining sections
         are added to the end in their original order

        :param order: Iterable of section names
        :return:
        """

        order_lc = [e.lower() for e in order]

        sections = OrderedDict( (k,self.sections[k]) for k in order_lc if k in self.sections)

        sections.update( (k,self.sections[k]) for k in self.sections.keys() if k not in order_lc)

        assert len(self.sections) == len(sections)

        self.sections = sections


Instruction:
Can you implement a function in Python 3 that
returns a list of terms that are in a particular section.

Response:
def find(self, term, value=False, section=None, _expand_derived=True, **kwargs):
        """Return a list of terms, possibly in a particular section. Use joined term notation, such as 'Root.Name' The kwargs arg is used to set term properties, all of which match returned terms, so ``name='foobar'`` will match terms that have a ``name`` property of ``foobar``

        :param term: The type of term to find, in fully-qulified notation, or use '*' for wild cards in either the parent or the record parts, such as 'Root.*', '*.Table' or '*.*'
        :param value: Select terms with a given value
        :param section: The name of the section in which to restrict the search
        :param kwargs:  See additional properties on which to match terms.

        """

        import itertools

        if kwargs:  # Look for terms with particular property values

            terms = self.find(term, value, section)

            found_terms = []

            for t in terms:
                if all(t.get_value(k) == v for k, v in kwargs.items()):
                    found_terms.append(t)

            return found_terms

        def in_section(term, section):

            if section is None:
                return True

            if term.section is None:
                return False

            if isinstance(section, (list, tuple)):
                return any(in_section(t, e) for e in section)
            else:
                return section.lower() == term.section.name.lower()

        # Try to replace the term with the list of its derived terms; that is, replace the super-class with all
        # of the derived classes, but only do this expansion once.
        if _expand_derived:
            try:
                try:
                    # Term is a string
                    term = list(self.derived_terms[term.lower()]) + [term]
                except AttributeError: # Term is hopefully a list
                    terms = []
                    for t in term:
                        terms.append(term)
                        for dt in self.derived_terms[t.lower()]:
                            terms.append(dt)
            except KeyError as e:
                pass

        # Find any of a list of terms
        if isinstance(term, (list, tuple)):
            return list(itertools.chain(*[self.find(e, value=value, section=section, _expand_derived=False) for e in term]))

        else:

            term = term.lower()

            found = []

            if not '.' in term:
                term = 'root.' + term

            if term.startswith('root.'):
                term_gen = self.terms # Just the root level terms
            else:
                term_gen = self.all_terms # All terms, root level and children.

            for t in term_gen:

                if t.join_lc == 'root.root':
                    continue

                assert t.section or t.join_lc == 'root.root' or t.join_lc == 'root.section', t

                if (t.term_is(term)
                    and in_section(t, section)
                    and (value is False or value == t.value)):
                    found.append(t)

            return found


Instruction:
Write a Python 3 script for
returning the first term in the list returning the default if no term is found

Response:
def get(self, term, default=None):
        """Return the first term, returning the default if no term is found"""
        v =  self.find_first(term)

        if not v:
            return default
        else:
            return v


Instruction:
Can you generate the documentation for the following Python 3 function
def get_value(self, term, default=None, section=None):
        term = self.find_first(term, value=False, section=section)

        if term is None:
            return default
        else:
            return term.value

Response: Return the first value of a term in the specified section.


Instruction:
How would you explain what the following Python 3 function does
def load_terms(self, terms):

        #if self.root and len(self.root.children) > 0:
        #    raise MetatabError("Can't run after adding terms to document.")

        for t in terms:

            t.doc = self

            if t.term_is('root.root'):
                if not self.root:
                    self.root = t
                    self.add_section(t)

                continue

            if t.term_is('root.section'):
                self.add_section(t)

            elif t.parent_term_lc == 'root':
                self.add_term(t)

            else:
                # These terms aren't added to the doc because they are attached to a
                # parent term that is added to the doc.
                assert t.parent is not None

        try:
            dd = terms.declare_dict

            self.decl_terms.update(dd['terms'])
            self.decl_sections.update(dd['sections'])

            self.super_terms.update(terms.super_terms())

            kf = lambda e: e[1]  # Sort on the value
            self.derived_terms ={ k:set( e[0] for e in g)
                                  for k, g in groupby(sorted(self.super_terms.items(), key=kf), kf)}

        except AttributeError as e:
            pass

        try:
            self.errors = terms.errors_as_dict()
        except AttributeError:
            self.errors = {}

        return self

Response: Create a builder from a sequence of terms.


Instruction:
Write a Python 3 script for
cleaning up some terms like ensuring that the name is a slug

Response:
def cleanse(self):
        """Clean up some terms, like ensuring that the name is a slug"""
        from .util import slugify

        self.ensure_identifier()

        try:
            self.update_name()
        except MetatabError:

            identifier = self['Root'].find_first('Root.Identifier')

            name = self['Root'].find_first('Root.Name')

            if name and name.value:
                name.value = slugify(name.value)
            elif name:
                name.value = slugify(identifier.value)
            else:
                self['Root'].get_or_new_term('Root.Name').value = slugify(identifier.value)


Instruction:
Write a Python 3 script for
generating the Root. Name term from DatasetName Version Origin TIme and Space and Create the Term Root. Name and Update the Name term.

Response:
def update_name(self, force=False, create_term=False, report_unchanged=True):
        """Generate the Root.Name term from DatasetName, Version, Origin, TIme and Space"""

        updates = []

        self.ensure_identifier()

        name_term = self.find_first('Root.Name')

        if not name_term:
            if create_term:
                name_term = self['Root'].new_term('Root.Name','')
            else:
                updates.append("No Root.Name, can't update name")
                return updates

        orig_name = name_term.value

        identifier = self.get_value('Root.Identifier')

        datasetname = self.get_value('Root.Dataset')

        if datasetname:

            name = self._generate_identity_name()

            if name != orig_name or force:
                name_term.value = name
                updates.append("Changed Name")
            else:
                if report_unchanged:
                    updates.append("Name did not change")

        elif not orig_name:

            if not identifier:
                updates.append("Failed to find DatasetName term or Identity term. Giving up")

            else:
                updates.append("Setting the name to the identifier")
                name_term.value = identifier

        elif orig_name == identifier:
            if report_unchanged:
                updates.append("Name did not change")

        else:
            # There is no DatasetName, so we can't gneerate name, and the Root.Name is not empty, so we should
            # not set it to the identity.
            updates.append("No Root.Dataset, so can't update the name")

        return updates


Instruction:
Can you tell what is the following Python 3 function doing
def as_dict(self, replace_value_names=True):

        # This function is a hack, due to confusion between the root of the document, which
        # should contain all terms, and the root section, which has only terms that are not
        # in another section. So, here we are taking the Root section, and adding all of the other
        # terms to it, as if it were also the root of the document tree.

        r = RootSectionTerm(doc=self)

        for s in self:  # Iterate over sections
            for t in s:  # Iterate over the terms in each section.
                r.terms.append(t)

        return r.as_dict(replace_value_names)

Response: Iterate link terms and convert to a dict


Instruction:
Make a summary of the following Python 3 code
def rows(self):

        for s_name, s in self.sections.items():

            # Yield the section header
            if s.name != 'Root':
                yield ['']  # Unecessary, but makes for nice formatting. Should actually be done just before write
                yield ['Section', s.value] + s.property_names

            # Yield all of the rows for terms in the section
            for row in s.rows:
                term, value = row

                term = term.replace('root.', '').title()

                try:
                    yield [term] + value
                except:
                    yield [term] + [value]

Response: Iterate over all of the rows in the archive


Instruction:
Write a Python 3 function that can
iterate over all of the terms in the hierarchy.

Response:
def all_terms(self):
        """Iterate over all of the terms. The self.terms property has only root level terms. This iterator
        iterates over all terms"""

        for s_name, s in self.sections.items():

            # Yield the section header
            if s.name != 'Root':
                yield s

            # Yield all of the rows for terms in the section
            for rterm in s:
                yield rterm
                for d in rterm.descendents:
                    yield d


Instruction:
Can you create a Python 3 function that
returns a CSV representation as a string

Response:
def as_csv(self):
        """Return a CSV representation as a string"""

        from io import StringIO

        s = StringIO()
        w = csv.writer(s)
        for row in self.rows:
            w.writerow(row)

        return s.getvalue()


Instruction:
Make a summary of the following Python 3 code
def as_lines(self):

        out_lines = []
        for t,v in self.lines:

            # Make the output prettier
            if t == 'Section':
                out_lines.append('')

            out_lines.append('{}: {}'.format(t,v if v is not None else '') )

        return '\n'.join(out_lines)

Response: Return a Lines representation as a string


Instruction:
Create a Python 3 function to
return a factory for creating QWidgetComponents.

Response:
def QWidgetComponentFactory(ui_file=None, *args, **kwargs):
    """
    Defines a class factory creating :class:`QWidgetComponent` classes using given ui file.

    :param ui_file: Ui file.
    :type ui_file: unicode
    :param \*args: Arguments.
    :type \*args: \*
    :param \*\*kwargs: Keywords arguments.
    :type \*\*kwargs: \*\*
    :return: QWidgetComponent class.
    :rtype: QWidgetComponent
    """

    class QWidgetComponent(foundations.ui.common.QWidget_factory(ui_file=ui_file)):

        """
        Defines the base class for **Manager** package QWidget Components.
        """

        component_activated = pyqtSignal()
        """
        This signal is emited by the :class:`QObjectComponent` class when the Component is activated.
        """

        component_deactivated = pyqtSignal()
        """
        This signal is emited by the :class:`QObjectComponent` class when the Component is deactivated.
        """

        component_initialized_ui = pyqtSignal()
        """
        This signal is emited by the :class:`QObjectComponent` class when the Component ui is initialized.
        """

        component_uninitialized_ui = pyqtSignal()
        """
        This signal is emited by the :class:`QObjectComponent` class when the Component ui is uninitialized.
        """

        def __init__(self, parent=None, name=None, *args, **kwargs):
            """
            Initializes the class.

            :param parent: Object parent.
            :type parent: QObject
            :param name: Component name.
            :type name: unicode
            :param \*args: Arguments.
            :type \*args: \*
            :param \*\*kwargs: Keywords arguments.
            :type \*\*kwargs: \*\*
            """

            LOGGER.debug("> Initializing '{0}()' class.".format(self.__class__.__name__))

            super(QWidgetComponent, self).__init__(parent, *args, **kwargs)

            # --- Setting class attributes. ---
            self.__name = None
            self.name = name

            self.__activated = False
            self.__initialized_ui = False
            self.__deactivatable = True

        @property
        def name(self):
            """
            Property for **self.__name** attribute.

            :return: self.__name.
            :rtype: unicode
            """

            return self.__name

        @name.setter
        @foundations.exceptions.handle_exceptions(AssertionError)
        def name(self, value):
            """
            Setter for **self.__name** attribute.

            :param value: Attribute value.
            :type value: unicode
            """

            if value is not None:
                assert type(value) is unicode, "'{0}' attribute: '{1}' type is not 'unicode'!".format(
                    "name", value)
            self.__name = value

        @name.deleter
        @foundations.exceptions.handle_exceptions(foundations.exceptions.ProgrammingError)
        def name(self):
            """
            Deleter for **self.__name** attribute.
            """

            raise foundations.exceptions.ProgrammingError("{0} | '{1}' attribute is not deletable!".format(
                self.__class__.__name__, "name"))

        @property
        def activated(self):
            """
            Property for **self.__activated** attribute.

            :return: self.__activated.
            :rtype: unicode
            """

            return self.__activated

        @activated.setter
        @foundations.exceptions.handle_exceptions(AssertionError)
        def activated(self, value):
            """
            Setter for **self.__activated** attribute.

            :param value: Attribute value.
            :type value: unicode
            """

            if value is not None:
                assert type(value) is bool, "'{0}' attribute: '{1}' type is not 'bool'!".format("activated", value)
                self.component_activated.emit() if value else self.component_deactivated.emit()
            self.__activated = value

        @activated.deleter
        @foundations.exceptions.handle_exceptions(foundations.exceptions.ProgrammingError)
        def activated(self):
            """
            Deleter for **self.__activated** attribute.
            """

            raise foundations.exceptions.ProgrammingError("{0} | '{1}' attribute is not deletable!".format(
                self.__class__.__name__, "activated"))

        @property
        def initialized_ui(self):
            """
            Property for **self.__initialized_ui** attribute.

            :return: self.__initialized_ui.
            :rtype: bool
            """

            return self.__initialized_ui

        @initialized_ui.setter
        @foundations.exceptions.handle_exceptions(AssertionError)
        def initialized_ui(self, value):
            """
            Setter for **self.__initialized_ui** attribute.

            :param value: Attribute value.
            :type value: bool
            """

            if value is not None:
                assert type(value) is bool, "'{0}' attribute: '{1}' type is not 'bool'!".format(
                    "initialized_ui", value)
                self.component_initialized_ui.emit() if value else self.component_uninitialized_ui.emit()
            self.__initialized_ui = value

        @initialized_ui.deleter
        @foundations.exceptions.handle_exceptions(foundations.exceptions.ProgrammingError)
        def initialized_ui(self):
            """
            Deleter for **self.__initialized_ui** attribute.
            """

            raise foundations.exceptions.ProgrammingError(
                "{0} | '{1}' attribute is not deletable!".format(self.__class__.__name__, "initialized_ui"))

        @property
        def deactivatable(self):
            """
            Property for **self.__deactivatable** attribute.

            :return: self.__deactivatable.
            :rtype: unicode
            """

            return self.__deactivatable

        @deactivatable.setter
        @foundations.exceptions.handle_exceptions(AssertionError)
        def deactivatable(self, value):
            """
            Setter for **self.__deactivatable** attribute.

            :param value: Attribute value.
            :type value: unicode
            """

            if value is not None:
                assert type(value) is bool, "'{0}' attribute: '{1}' type is not 'bool'!".format("deactivatable", value)
            self.__deactivatable = value

        @deactivatable.deleter
        @foundations.exceptions.handle_exceptions(foundations.exceptions.ProgrammingError)
        def deactivatable(self):
            """
            Deleter for **self.__deactivatable** attribute.
            """

            raise foundations.exceptions.ProgrammingError("{0} | '{1}' attribute is not deletable!".format(
                self.__class__.__name__, "deactivatable"))

        @foundations.exceptions.handle_exceptions(NotImplementedError)
        def activate(self):
            """
            Sets Component activation state.

            :return: Method success.
            :rtype: bool
            """

            raise NotImplementedError("{0} | '{1}' must be implemented by '{2}' subclasses!".format(
                self.__class__.__name__, self.activate.__name__, self.__class__.__name__))

        @foundations.exceptions.handle_exceptions(NotImplementedError)
        def deactivate(self):
            """
            Unsets Component activation state.

            :return: Method success.
            :rtype: bool
            """

            raise NotImplementedError("{0} | '{1}' must be implemented by '{2}' subclasses!".format(
                self.__class__.__name__, self.deactivate.__name__, self.__class__.__name__))

        @foundations.exceptions.handle_exceptions(NotImplementedError)
        def initialize_ui(self):
            """
            Initializes the Component ui.
            """

            raise NotImplementedError("{0} | '{1}' must be implemented by '{2}' subclasses!".format(
                self.__class__.__name__, self.deactivate.__name__, self.__class__.__name__))

        @foundations.exceptions.handle_exceptions(NotImplementedError)
        def add_widget(self):
            """
            Adds the Component Widget ui.
            """

            raise NotImplementedError("{0} | '{1}' must be implemented by '{2}' subclasses!".format(
                self.__class__.__name__, self.deactivate.__name__, self.__class__.__name__))

        @foundations.exceptions.handle_exceptions(NotImplementedError)
        def remove_widget(self):
            """
            Removes the Component Widget ui.
            """

            raise NotImplementedError("{0} | '{1}' must be implemented by '{2}' subclasses!".format(
                self.__class__.__name__, self.deactivate.__name__, self.__class__.__name__))

        @foundations.exceptions.handle_exceptions(NotImplementedError)
        def uninitialize_ui(self):
            """
            Uninitializes the Component ui.
            """

            raise NotImplementedError("{0} | '{1}' must be implemented by '{2}' subclasses!".format(
                self.__class__.__name__, self.deactivate.__name__, self.__class__.__name__))

    return QWidgetComponent


Instruction:
Create a Python 3 function for
removing volume node if it follows a journal logically in the tree hierarchy.

Response:
def _restructure_if_volume_follows_journal(left, right):
    """Remove volume node if it follows a journal logically in the tree hierarchy.

    Args:
        left (ast.ASTElement): The journal KeywordOp node.
        right (ast.ASTElement): The rest of the tree to be restructured.

    Return:
        (ast.ASTElement): The restructured tree, with the volume node removed.

    Notes:
        This happens to support queries like "journal Phys.Rev. and vol d85". Appends the value of KeywordOp with
        Keyword 'volume' and discards 'volume' KeywordOp node from the tree.
    """
    def _get_volume_keyword_op_and_remaining_subtree(right_subtree):
        if isinstance(right_subtree, NotOp) and isinstance(right_subtree.op, KeywordOp) \
                and right_subtree.op.left == Keyword('volume'):
            return None, None

        elif isinstance(right_subtree, AndOp) and isinstance(right_subtree.left, NotOp) \
                and isinstance(right_subtree.left.op, KeywordOp) and right_subtree.left.op.left == Keyword('volume'):
            return None, right_subtree.right

        elif isinstance(right_subtree, KeywordOp) and right_subtree.left == Keyword('volume'):
            return right_subtree, None

        elif isinstance(right_subtree, AndOp) and right_subtree.left.left == Keyword('volume'):
            return right_subtree.left, right_subtree.right

    journal_value = left.right.value

    volume_and_remaining_subtree = _get_volume_keyword_op_and_remaining_subtree(right)
    if not volume_and_remaining_subtree:
        return

    volume_node, remaining_subtree = volume_and_remaining_subtree
    if volume_node:
        left.right.value = ','.join([journal_value, volume_node.right.value])

    return AndOp(left, remaining_subtree) if remaining_subtree else left


Instruction:
How would you implement a function in Python 3 that
converts a SimpleValueBooleanQuery to AndOp queries.

Response:
def _convert_simple_value_boolean_query_to_and_boolean_queries(tree, keyword):
    """Chain SimpleValueBooleanQuery values into chained AndOp queries with the given current Keyword."""

    def _create_operator_node(value_node):
        """Creates a KeywordOp or a ValueOp node."""
        base_node = value_node.op if isinstance(value_node, NotOp) else value_node
        updated_base_node = KeywordOp(keyword, base_node) if keyword else ValueOp(base_node)

        return NotOp(updated_base_node) if isinstance(value_node, NotOp) else updated_base_node

    def _get_bool_op_type(bool_op):
        return AndOp if isinstance(bool_op, And) else OrOp

    new_tree_root = _get_bool_op_type(tree.bool_op)(None, None)
    current_tree = new_tree_root
    previous_tree = tree

    while True:  # Walk down the tree while building the new AndOp queries subtree.
        current_tree.left = _create_operator_node(previous_tree.left)

        if not isinstance(previous_tree.right, SimpleValueBooleanQuery):
            current_tree.right = _create_operator_node(previous_tree.right)
            break

        previous_tree = previous_tree.right
        current_tree.right = _get_bool_op_type(previous_tree.bool_op)(None, None)
        current_tree = current_tree.right

    return new_tree_root


Instruction:
Can you tell what is the following Python 3 function doing
def visit_boolean_query(self, node):
        left = node.left.accept(self)
        right = node.right.accept(self)

        is_journal_keyword_op = isinstance(left, KeywordOp) and left.left == Keyword('journal')

        if is_journal_keyword_op:
            journal_and_volume_conjunction = _restructure_if_volume_follows_journal(left, right)

            if journal_and_volume_conjunction:
                return journal_and_volume_conjunction

        return AndOp(left, right) if isinstance(node.bool_op, And) else OrOp(left, right)

Response: Convert BooleanRule into AndOp or OrOp nodes.


Instruction:
Can you implement a function in Python 3 that
visits the SimpleValueBooleanQuery node and replace the node with the actual node type.

Response:
def visit_simple_value_boolean_query(self, node):
        """
        Visits only the children of :class:`SimpleValueBooleanQuery` without substituting the actual node type.

        Notes:
            Defer conversion from :class:`SimpleValueBooleanQuery` to AndOp or OrOp.
            This transformation needs to occur higher in the tree, so that we don't lose the information that this is a
            boolean query among terminals and thus the associative rule needs to be applied if we reached here from a
            keyword query, or a conversion from :class:`SimpleValueBooleanQuery` to :class:`AndOp` or :class:`OrOp`,
            otherwise.
        """
        node.left, node.right = node.left.accept(self), node.right.accept(self)
        return node


Instruction:
How would you explain what the following Python 3 function does
def visit_spires_keyword_query(self, node):
        keyword = node.left.accept(self)
        value = node.right.accept(self)

        if isinstance(value, SimpleValueBooleanQuery):
            return _convert_simple_value_boolean_query_to_and_boolean_queries(value, keyword)

        return KeywordOp(keyword, value)

Response: Visit a SpiresKeywordQuery node by returning a KeywordOp.


Instruction:
Write a Python 3 function that can
visit an InvenioKeywordQuery node by returning a KeywordOp.

Response:
def visit_invenio_keyword_query(self, node):
        """Transform an :class:`InvenioKeywordQuery` into a :class:`KeywordOp`.

        Notes:
            In case the value being a :class:`SimpleValueBooleanQuery`, the subtree is transformed to chained
            :class:`AndOp` queries containing :class:`KeywordOp`, whose keyword is the keyword of the current node and
            values, all the :class:`SimpleValueBooleanQuery` values (either :class:`SimpleValues` or
            :class:`SimpleValueNegation`.)
        """
        try:
            keyword = node.left.accept(self)
        except AttributeError:
            # The keywords whose values aren't an InspireKeyword are simple strings.
            keyword = Keyword(node.left)

        value = node.right.accept(self)

        if isinstance(value, SimpleValueBooleanQuery):
            return _convert_simple_value_boolean_query_to_and_boolean_queries(value, keyword)

        return KeywordOp(keyword, value)


Instruction:
Implement a Python 3 function for
converting a node to one of ExactMatch PartialMatch and Regex Value nodes.

Response:
def visit_complex_value(self, node):
        """Convert :class:`ComplexValue` to one of ExactMatch, PartialMatch and Regex Value nodes."""
        if node.value.startswith(ComplexValue.EXACT_VALUE_TOKEN):
            value = node.value.strip(ComplexValue.EXACT_VALUE_TOKEN)
            return ExactMatchValue(value)

        elif node.value.startswith(ComplexValue.PARTIAL_VALUE_TOKEN):
            value = node.value.strip(ComplexValue.PARTIAL_VALUE_TOKEN)
            return PartialMatchValue(value, True if ast.GenericValue.WILDCARD_TOKEN in value else False)

        elif node.value.startswith(ComplexValue.REGEX_VALUE_TOKEN):
            return RegexValue(node.value.strip(ComplexValue.REGEX_VALUE_TOKEN))
        else:
            # Covering the case where ComplexValue supports more than ExactMatch, PartialMatch and Regex values.
            msg = self.__class__.__name__ + ': Unrecognized complex value'
            try:
                msg += ' lookahead token: "' + node.value[0] + '"'
            except IndexError:
                msg += ': \"' + repr(node.value) + '"'
            msg += '.\nUsing simple value instead: "' + node.value + '".'
            logger.warn(msg)
            return ast.Value(node.value)


Instruction:
Can you generate a brief explanation for the following Python 3 code
async def stop(self, force: bool=False) -> None:
        '''Cancel the task if it hasn't yet started, or tell it to
        gracefully stop running if it has.'''

        Log.debug('stopping task %s', self.name)
        self.running = False

        if force:
            self.task.cancel()

Response: Cancel the task if it hasn t yet started.


Instruction:
How would you code a function in Python 3 to
recognize whether the author name contains full name parts and not initials or only lastname.

Response:
def author_name_contains_fullnames(author_name):
    """Recognizes whether the name contains full name parts and not initials or only lastname.

    Returns:
          bool: True if name has only full name parts, e.g. 'Ellis John', False otherwise. So for example, False is
            returned for 'Ellis, J.' or 'Ellis'.
    """
    def _is_initial(name_part):
        return len(name_part) == 1 or u'.' in name_part

    parsed_name = ParsedName(author_name)

    if len(parsed_name) == 1:
        return False
    elif any([_is_initial(name_part) for name_part in parsed_name]):
        return False

    return True


Instruction:
Can you create a Python 3 function that
detects whether the name variation consists only from initials.

Response:
def _name_variation_has_only_initials(name):
    """Detects whether the name variation consists only from initials."""
    def _is_initial(name_variation):
        return len(name_variation) == 1 or u'.' in name_variation

    parsed_name = ParsedName.loads(name)

    return all([_is_initial(name_part) for name_part in parsed_name])


Instruction:
Given the following Python 3 function, write the documentation
def generate_minimal_name_variations(author_name):
    parsed_name = ParsedName.loads(unidecode(author_name))

    if len(parsed_name) > 1:
        lastnames = parsed_name.last.replace('-', ' ')

        non_lastnames = ' '.join(
            parsed_name.first_list + parsed_name.suffix_list
        )
        # Strip extra whitespace added if any of middle_list and suffix_list are empty.
        non_lastnames = non_lastnames.strip().replace('-', ' ')

        # Adding into a set first, so as to drop identical name variations.
        return list({
            name_variation.lower()
            for name_variation
            in [
                lastnames + ' ' + non_lastnames,
                lastnames + ' ' + non_lastnames[0],
                non_lastnames + ' ' + lastnames,
                non_lastnames + ' ' + lastnames[0],
            ]
            if not _name_variation_has_only_initials(name_variation)
        })
    else:
        return [parsed_name.dumps().replace('-', ' ').lower()]

Response: Generates a small number of name variations.


Instruction:
Implement a Python 3 function for
truncating wildcard from date parts.

Response:
def _truncate_wildcard_from_date(date_value):
    """Truncate wildcard from date parts.

    Returns:
        (str) The truncated date.

    Raises:
        ValueError, on either unsupported date separator (currently only ' ' and '-' are supported), or if there's a
        wildcard in the year.

    Notes:
        Either whole date part is wildcard, in which we ignore it and do a range query on the
        remaining parts, or some numbers are wildcards, where again, we ignore this part.
    """
    if ' ' in date_value:
        date_parts = date_value.split(' ')
    elif '-' in date_value:
        date_parts = date_value.split('-')
    else:
        # Either unsupported separators or wildcard in year, e.g. '201*'.
        raise ValueError("Erroneous date value: %s.", date_value)

    if GenericValue.WILDCARD_TOKEN in date_parts[-1]:
        del date_parts[-1]

    return '-'.join(date_parts)


Instruction:
How would you code a function in Python 3 to
truncate the given date value according to the given date field.

Response:
def _truncate_date_value_according_on_date_field(field, date_value):
    """Truncates date value (to year only) according to the given date field.

    Args:
        field (unicode): The field for which the date value will be used to query on.
        date_value (str): The date value that is going to be truncated to its year.

    Returns:
        PartialDate: The possibly truncated date, on success. None, otherwise.

    Notes:
        In case the fieldname is in `ES_MAPPING_HEP_DATE_ONLY_YEAR`, then the date is normalized and then only its year
        value is used. This is needed for ElasticSearch to be able to do comparisons on dates that have only year, which
        fails if being queried with a date with more .
    """
    try:
        partial_date = PartialDate.parse(date_value)
    except ValueError:
        return None

    if field in ES_MAPPING_HEP_DATE_ONLY_YEAR:
        truncated_date = PartialDate.from_parts(partial_date.year)
    else:
        truncated_date = partial_date

    return truncated_date


Instruction:
Here you have a function in Python 3, explain what it does
def _get_next_date_from_partial_date(partial_date):
    relativedelta_arg = 'years'

    if partial_date.month:
        relativedelta_arg = 'months'
    if partial_date.day:
        relativedelta_arg = 'days'

    next_date = parse(partial_date.dumps()) + relativedelta(**{relativedelta_arg: 1})
    return PartialDate.from_parts(
        next_date.year,
        next_date.month if partial_date.month else None,
        next_date.day if partial_date.day else None
    )

Response: Calculates the next date from the given partial date.


Instruction:
Here you have a function in Python 3, explain what it does
def _get_proper_elastic_search_date_rounding_format(partial_date):
    es_date_math_unit = ES_DATE_MATH_ROUNDING_YEAR

    if partial_date.month:
        es_date_math_unit = ES_DATE_MATH_ROUNDING_MONTH
    if partial_date.day:
        es_date_math_unit = ES_DATE_MATH_ROUNDING_DAY

    return es_date_math_unit

Response: Returns the proper ES date math unit according to the resolution of the partial_date.


Instruction:
Write a Python 3 script for
updating the operator - value pairs for a given field.

Response:
def update_date_value_in_operator_value_pairs_for_fieldname(field, operator_value_pairs):
    """Updates (operator, date value) pairs by normalizing the date value according to the given field.

    Args:
        field (unicode): The fieldname for which the operator-value pairs are being generated.
        operator_value_pairs (dict): ES range operator {'gt', 'gte', 'lt', 'lte'} along with a value.
            Additionally, if the operator is ``ES_RANGE_EQ_OPERATOR``, then it is indicated that the method should
            generate both a lower and an upper bound operator-value pairs, with the given date_value.

    Notes:
        On a ``ValueError`` an empty operator_value_pairs is returned.
    """
    updated_operator_value_pairs = {}
    for operator, value in operator_value_pairs.items():
        modified_date = _truncate_date_value_according_on_date_field(field, value)
        if not modified_date:
            return {}

        if operator == ES_RANGE_EQ_OPERATOR:
            updated_operator_value_pairs['gte'] = \
                modified_date.dumps() + _get_proper_elastic_search_date_rounding_format(modified_date)

            next_date = _get_next_date_from_partial_date(modified_date)
            updated_operator_value_pairs['lt'] = \
                next_date.dumps() + _get_proper_elastic_search_date_rounding_format(next_date)
        else:
            updated_operator_value_pairs[operator] = \
                modified_date.dumps() + _get_proper_elastic_search_date_rounding_format(modified_date)

    return updated_operator_value_pairs


Instruction:
Explain what the following Python 3 code does
def generate_match_query(field, value, with_operator_and):
    parsed_value = None
    try:
        parsed_value = json.loads(value.lower())
    except (ValueError, TypeError, AttributeError):
        # Catch all possible exceptions
        # we are not interested if they will appear
        pass

    if isinstance(value, bool):
        return {'match': {field: value}}
    elif isinstance(parsed_value, bool):
        return {'match': {field: value.lower()}}

    if with_operator_and:
        return {
            'match': {
                field: {
                    'query': value,
                    'operator': 'and'
                }
            }
        }

    return {'match': {field: value}}

Response: Generates a match query for the given ES field and value.


Instruction:
Create a Python 3 function for
wrapping a query into a nested if the fields within the query are nested.

Response:
def wrap_query_in_nested_if_field_is_nested(query, field, nested_fields):
    """Helper for wrapping a query into a nested if the fields within the query are nested

    Args:
        query : The query to be wrapped.
        field : The field that is being queried.
        nested_fields : List of fields which are nested.
    Returns:
        (dict): The nested query
    """
    for element in nested_fields:
        match_pattern = r'^{}.'.format(element)
        if re.match(match_pattern, field):
            return generate_nested_query(element, query)

    return query


Instruction:
Create a Python 3 function to
return True if the difference between date and any value in dates is less than or equal to limit.

Response:
def is_within_limits(self, limit, date, dates):
        """
        Returns True if the difference between date and any value in dates
        is less than or equal to limit.
        """
        return any((self.second_diff(date, d) <= limit for d in dates))


Instruction:
Can you generate the documentation for the following Python 3 function
def emit_tree_format(tree, verbose=False):
    if verbose:
        print("Converting: " + repr(tree))
    ret_str = __recursive_formatter(tree)
    return ret_str

Response: Returns a tree - like representation of a parse tree


Instruction:
Can you write a function in Python 3 where it
calculates an exponential backoff for retry attempts with a small amount of jitter.

Response:
def calculate_retry_delay(attempt, max_delay=300):
    """Calculates an exponential backoff for retry attempts with a small
    amount of jitter."""
    delay = int(random.uniform(2, 4) ** attempt)
    if delay > max_delay:
        # After reaching the max delay, stop using expontential growth
        # and keep the delay nearby the max.
        delay = int(random.uniform(max_delay - 20, max_delay + 20))
    return delay


Instruction:
Write a Python 3 script to
parse a given query string and returns a multi_match query.

Response:
def parse_query(query_str):
    """
    Drives the whole logic, by parsing, restructuring and finally, generating an ElasticSearch query.

    Args:
        query_str (six.text_types): the given query to be translated to an ElasticSearch query

    Returns:
        six.text_types: Return an ElasticSearch query.

    Notes:
        In case there's an error, an ElasticSearch `multi_match` query is generated with its `query` value, being the
        query_str argument.
    """
    def _generate_match_all_fields_query():
        # Strip colon character (special character for ES)
        stripped_query_str = ' '.join(query_str.replace(':', ' ').split())
        return {'multi_match': {'query': stripped_query_str, 'fields': ['_all'], 'zero_terms_query': 'all'}}

    if not isinstance(query_str, six.text_type):
        query_str = six.text_type(query_str.decode('utf-8'))

    logger.info('Parsing: "' + query_str + '\".')

    parser = StatefulParser()
    rst_visitor = RestructuringVisitor()
    es_visitor = ElasticSearchVisitor()

    try:
        unrecognized_text, parse_tree = parser.parse(query_str, Query)

        if unrecognized_text:  # Usually, should never happen.
            msg = 'Parser returned unrecognized text: "' + unrecognized_text + \
                  '" for query: "' + query_str + '".'

            if query_str == unrecognized_text and parse_tree is None:
                # Didn't recognize anything.
                logger.warn(msg)
                return _generate_match_all_fields_query()
            else:
                msg += 'Continuing with recognized parse tree.'
            logger.warn(msg)

    except SyntaxError as e:
        logger.warn('Parser syntax error (' + six.text_type(e) + ') with query: "' + query_str +
                    '". Continuing with a match_all with the given query.')
        return _generate_match_all_fields_query()

    # Try-Catch-all exceptions for visitors, so that search functionality never fails for the user.
    try:
        restructured_parse_tree = parse_tree.accept(rst_visitor)
        logger.debug('Parse tree: \n' + emit_tree_format(restructured_parse_tree))

    except Exception as e:
        logger.exception(
            RestructuringVisitor.__name__ + " crashed" + (": " + six.text_type(e) + ".") if six.text_type(e) else '.'
        )
        return _generate_match_all_fields_query()

    try:
        es_query = restructured_parse_tree.accept(es_visitor)
    except Exception as e:
        logger.exception(
            ElasticSearchVisitor.__name__ + " crashed" + (": " + six.text_type(e) + ".") if six.text_type(e) else '.'
        )
        return _generate_match_all_fields_query()

    if not es_query:
        # Case where an empty query was generated (i.e. date query with malformed date, e.g. "d < 200").
        return _generate_match_all_fields_query()

    return es_query


Instruction:
Create a Python 3 function for
returning a running Task object matching the given name or class.

Response:
def task(self, name_or_class: Any) -> Task:
        '''Return a running Task object matching the given name or class.'''

        if name_or_class in self.all_tasks:
            return self.all_tasks[name_or_class]

        try:
            return self.all_tasks.get(name_or_class.__class__.__name__, None)

        except AttributeError:
            return None


Instruction:
Implement a Python 3 function for
initializing configuration and start tasks.

Response:
async def init(self) -> None:
        '''Initialize configuration and start tasks.'''

        self.stats = await self.insert(self.stats)
        self.configuration = await self.insert(self.configuration)

        if not self.executor:
            try:
                max_workers = self.config.get('executor_workers')
            except Exception:
                max_workers = None

            self.executor = ThreadPoolExecutor(max_workers=max_workers)

        for task in self.initial_tasks:
            await self.insert(task)

        self.monitor = asyncio.ensure_future(self.monitor_tasks())
        self.counters['alive_since'] = time.time()


Instruction:
Can you generate the documentation for the following Python 3 function
async def insert(self, task: Task) -> None:
        '''Insert the given task class into the Tasky event loop.'''

        if not isinstance(task, Task):
            task = task()

        if task.name not in self.all_tasks:
            task.tasky = self
            self.all_tasks[task.name] = task

            await task.init()

        elif task != self.all_tasks[task.name]:
            raise Exception('Duplicate task %s' % task.name)

        if task.enabled:
            task.task = asyncio.ensure_future(self.start_task(task))
            self.running_tasks.add(task)
        else:
            task.task = None

        return task

Response: Insert the given task class into the Tasky event loop.


Instruction:
Write a Python 3 script to
execute an arbitrary function outside the event loop using a shared Executor.

Response:
async def execute(self, fn, *args, **kwargs) -> None:
        '''Execute an arbitrary function outside the event loop using
        a shared Executor.'''

        fn = functools.partial(fn, *args, **kwargs)
        return await self.loop.run_in_executor(self.executor, fn)


Instruction:
Implement a function in Python 3 to
execute the tasky event loop until terminated.

Response:
def run_forever(self) -> None:
        '''Execute the tasky/asyncio event loop until terminated.'''

        Log.debug('running event loop until terminated')
        asyncio.ensure_future(self.init())
        self.loop.run_forever()
        self.loop.close()


Instruction:
Explain what the following Python 3 code does
def run_until_complete(self) -> None:
        '''Execute the tasky/asyncio event loop until all tasks finish.'''

        Log.debug('running event loop until all tasks completed')
        self.terminate_on_finish = True
        asyncio.ensure_future(self.init())
        self.loop.run_forever()
        self.loop.close()

Response: Execute the tasky event loop until all tasks finish.


Instruction:
Write a Python 3 script for
executing the tasky event loop for duration seconds.

Response:
def run_for_time(self, duration: float=10.0) -> None:
        '''Execute the tasky/asyncio event loop for `duration` seconds.'''

        Log.debug('running event loop for %.1f seconds', duration)
        try:
            asyncio.ensure_future(self.init())
            self.loop.run_until_complete(asyncio.sleep(duration))
            self.terminate()
            self.loop.run_forever()

        except RuntimeError as e:
            if not e.args[0].startswith('Event loop stopped'):
                raise

        finally:
            self.loop.close()


Instruction:
How would you implement a function in Python 3 that
terminates all scheduled and executing tasks.

Response:
def terminate(self, *, force: bool=False, timeout: float=30.0,
                  step: float=1.0) -> None:
        '''Stop all scheduled and/or executing tasks, first by asking nicely,
        and then by waiting up to `timeout` seconds before forcefully stopping
        the asyncio event loop.'''

        if isinstance(self.monitor, asyncio.Future):
            Log.debug('cancelling task monitor')
            self.monitor.cancel()

        Log.debug('stopping tasks')
        for task in list(self.running_tasks):
            if task.task.done():
                Log.debug('task %s already stopped', task.name)
                self.running_tasks.discard(task)
            else:
                Log.debug('asking %s to stop', task.name)
                asyncio.ensure_future(task.stop(force=force))

        if timeout > 0 and (self.monitor or self.running_tasks):
            Log.debug('waiting %.1f seconds for remaining tasks (%d)...',
                      timeout, len(self.running_tasks))

            timeout -= step
            fn = functools.partial(self.terminate, force=force,
                                   timeout=timeout, step=step)
            return self.loop.call_later(step, fn)

        if timeout > 0:
            Log.debug('all tasks completed, stopping event loop')
        else:
            Log.debug('timed out waiting for tasks, stopping event loop')

        self.loop.stop()


Instruction:
Implement a Python 3 function for
initializing the task queue it for execution add the done callback and keep track of it for when tasks need to be stopped.

Response:
async def start_task(self, task: Task) -> None:
        '''Initialize the task, queue it for execution, add the done callback,
        and keep track of it for when tasks need to be stopped.'''

        try:
            Log.debug('task %s starting', task.name)
            before = time.time()
            task.counters['last_run'] = before

            task.running = True
            self.running_tasks.add(task)
            await task.run_task()
            Log.debug('task %s completed', task.name)

        except CancelledError:
            Log.debug('task %s cancelled', task.name)

        except Exception:
            Log.exception('unhandled exception in task %s', task.name)

        finally:
            self.running_tasks.discard(task)
            task.running = False
            task.task = None

            after = time.time()
            total = after - before
            task.counters['last_completed'] = after
            task.counters['duration'] = total


Instruction:
Implement a function in Python 3 to
monitor all known tasks for run state.

Response:
async def monitor_tasks(self, interval: float=1.0) -> None:
        '''Monitor all known tasks for run state.  Ensure that enabled tasks
        are running, and that disabled tasks are stopped.'''

        Log.debug('monitor running')
        while True:
            try:
                await asyncio.sleep(interval)

                for name, task in self.all_tasks.items():
                    if self.terminate_on_finish:
                        if task in self.running_tasks and task.running:
                            await task.stop()

                    elif task.enabled:
                        if task not in self.running_tasks:
                            Log.debug('task %s enabled, restarting', task.name)
                            await self.insert(task)

                    else:
                        if task in self.running_tasks:
                            Log.debug('task %s disabled, stopping', task.name)
                            await task.stop()

                if self.terminate_on_finish and not self.running_tasks:
                    Log.debug('all tasks completed, terminating')
                    break

            except CancelledError:
                Log.debug('monitor cancelled')
                break

            except Exception:
                Log.exception('monitoring exception')

        self.monitor = None
        self.loop.call_later(0, self.terminate)


Instruction:
Given the following Python 3 function, write the documentation
def exception(self, loop: asyncio.BaseEventLoop, context: dict) -> None:
        '''Log unhandled exceptions from anywhere in the event loop.'''

        Log.error('unhandled exception: %s', context['message'])
        Log.error('%s', context)
        if 'exception' in context:
            Log.error('  %s', context['exception'])

Response: Log unhandled exceptions from anywhere in the event loop.


Instruction:
How would you explain what the following Python 3 function does
def sigint(self) -> None:
        '''Handle the user pressing Ctrl-C by stopping tasks nicely at first,
        then forcibly upon further presses.'''

        if self.stop_attempts < 1:
            Log.info('gracefully stopping tasks')
            self.stop_attempts += 1
            self.terminate()

        elif self.stop_attempts < 2:
            Log.info('forcefully cancelling tasks')
            self.stop_attempts += 1
            self.terminate(force=True)

        else:
            Log.info('forcefully stopping event loop')
            self.loop.stop()

Response: Handle the user pressing Ctrl - C by stopping tasks nicely at first and forcibly upon further presses.


Instruction:
How would you implement a function in Python 3 that
handles SIGTERM from the system by stopping tasks gracefully.

Response:
def sigterm(self) -> None:
        '''Handle SIGTERM from the system by stopping tasks gracefully.
        Repeated signals will be ignored while waiting for tasks to finish.'''

        if self.stop_attempts < 1:
            Log.info('received SIGTERM, gracefully stopping tasks')
            self.stop_attempts += 1
            self.terminate()

        else:
            Log.info('received SIGTERM, bravely waiting for tasks')


Instruction:
Write a Python 3 function for
checking if terminal token is a keyword after lower - casing it.

Response:
def parse(cls, parser, text, pos):
        """Checks if terminal token is a keyword after lower-casing it."""
        match = cls.regex.match(text)
        if match:
            # Check if match is is not in the grammar of the specific keyword class.
            if match.group(0).lower() not in cls.grammar:
                result = text, SyntaxError(repr(match.group(0)) + " is not a member of " + repr(cls.grammar))
            else:
                result = text[len(match.group(0)):], cls(match.group(0))
        else:
            result = text, SyntaxError("expecting " + repr(cls.__name__))
        return result


Instruction:
Can you generate the documentation for the following Python 3 function
def parse(cls, parser, text, pos):
        try:
            remaining_text, keyword = parser.parse(text, cls.grammar)
            if keyword.lower() == 'texkey':
                parser._parsing_texkey_expression = True
            return remaining_text, InspireKeyword(keyword)
        except SyntaxError as e:
            parser._parsing_texkey_expression = False
            return text, e

Response:
Parse InspireKeyword.

        If the keyword is `texkey`, enable the parsing texkey expression flag, since its value contains ':' which
        normally isn't allowed.


Instruction:
Write a Python 3 script to
parse a terminal token that doesn t contain parentheses nor colon symbol.

Response:
def parse_terminal_token(cls, parser, text):
        """Parses a terminal token that doesn't contain parentheses nor colon symbol.

        Note:
            Handles a special case of tokens where a ':' is needed (for `texkey` queries).

            If we're parsing text not in parentheses, then some DSL keywords (e.g. And, Or, Not, defined above) should
            not be recognized as terminals, thus we check if they are in the Keywords table (namespace like structure
            handled by PyPeg).
            This is done only when we are not parsing a parenthesized SimpleValue.

            Also, helps in supporting more implicit-and queries cases (last two checks).
        """
        token_regex = cls.token_regex
        if parser._parsing_texkey_expression:
            token_regex = cls.texkey_token_regex
            parser._parsing_texkey_expression = False

        match = token_regex.match(text)
        if match:
            matched_token = match.group(0)

            # Check if token is a DSL keyword. Disable this check in the case where the parser isn't parsing a
            # parenthesized terminal.
            if not parser._parsing_parenthesized_terminal and matched_token.lower() in Keyword.table:
                return text, SyntaxError("found DSL keyword: " + matched_token)

            remaining_text = text[len(matched_token):]

            # Attempt to recognize whether current terminal is followed by a ":", which definitely signifies that
            # we are parsing a keyword, and we shouldn't.
            if cls.starts_with_colon.match(remaining_text):
                return text, \
                       SyntaxError("parsing a keyword (token followed by \":\"): \"" + repr(matched_token) + "\"")

            # Attempt to recognize whether current terminal is a non shortened version of Inspire keywords. This is
            # done for supporting implicit-and in case of SPIRES style keyword queries. Using the non shortened version
            # of the keywords, makes this recognition not eager.
            if not parser._parsing_parenthesized_simple_values_expression \
                    and matched_token in INSPIRE_KEYWORDS_SET:
                return text, SyntaxError("parsing a keyword (non shortened INSPIRE keyword)")

            result = remaining_text, matched_token
        else:
            result = text, SyntaxError("expecting match on " + repr(cls.token_regex.pattern))
        return result


Instruction:
Write a Python 3 script to
imitate parsing a list grammar.

Response:
def parse(cls, parser, text, pos):
        """Imitates parsing a list grammar.

        Specifically, this
        grammar = [
            SimpleValueUnit.date_specifiers_regex,
            SimpleValueUnit.arxiv_token_regex,
            SimpleValueUnit.token_regex,
            SimpleValueUnit.parenthesized_token_grammar
        ].

        Parses plaintext which matches date specifiers or arxiv_identifier syntax, or is comprised of either 1) simple
        terminal (no parentheses) or 2) a parenthesized SimpleValue.

        For example, "e(+)" will be parsed in two steps, first, "e" token will be recognized and then "(+)", as a
        parenthesized SimpleValue.
        """
        found = False

        # Attempt to parse date specifier
        match = cls.date_specifiers_regex.match(text)
        if match:
            remaining_text, token, found = text[len(match.group(0)):], match.group(0), True
        else:
            # Attempt to parse arxiv identifier
            match = cls.arxiv_token_regex.match(text)
            if match:
                remaining_text, token, found = text[len(match.group()):], match.group(2), True
            else:
                # Attempt to parse a terminal token
                remaining_text, token = SimpleValueUnit.parse_terminal_token(parser, text)
                if type(token) != SyntaxError:
                    found = True
                else:
                    # Attempt to parse a terminal with parentheses
                    try:
                        # Enable parsing a parenthesized terminal so that we can accept {+, -, |} as terminals.
                        parser._parsing_parenthesized_terminal = True
                        remaining_text, token = parser.parse(text, cls.parenthesized_token_grammar, pos)

                        found = True
                    except SyntaxError:
                        pass
                    except GrammarValueError:
                        raise
                    except ValueError:
                        pass
                    finally:
                        parser._parsing_parenthesized_terminal = False

        if found:
            result = remaining_text, SimpleValueUnit(token)
        else:
            result = text, SyntaxError("expecting match on " + cls.__name__)

        return result


Instruction:
Can you generate a brief explanation for the following Python 3 code
def unconsume_and_reconstruct_input(remaining_text, recognized_tokens, complex_value_idx):
        # Default slicing index: i.e. at most 3 elements will be unconsumed, Keyword, Whitespace and ComplexValue.
        slicing_start_idx = 2

        # Check whether the 3rd element from the end is an InspireKeyword. If not, a Value query with ComplexValue
        # was consumed.
        if not INSPIRE_PARSER_KEYWORDS.get(recognized_tokens[complex_value_idx - slicing_start_idx].value, None):
            slicing_start_idx = 1

        reconstructed_terminals = recognized_tokens[:complex_value_idx - slicing_start_idx]
        reconstructed_text = '{} {}'.format(
            ''.join([token.value for token in recognized_tokens[complex_value_idx - slicing_start_idx:]]),
            remaining_text
        )
        return reconstructed_text, reconstructed_terminals

Response: Reconstructs input in case of consuming a keyword query or a value query with ComplexValue as value.


Instruction:
Can you write a function in Python 3 where it
parses the current state of the object.

Response:
def parse(cls, parser, text, pos):
        """Using our own parse to enable the flag below."""
        try:
            parser._parsing_parenthesized_simple_values_expression = True
            remaining_text, recognized_tokens = parser.parse(text, cls.grammar)
            return remaining_text, recognized_tokens
        except SyntaxError as e:
            return text, e
        finally:
            parser._parsing_parenthesized_simple_values_expression = False


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _generate_fieldnames_if_bai_query(self, node_value, bai_field_variation, query_bai_field_if_dots_in_name):
        if bai_field_variation not in (FieldVariations.search, FieldVariations.raw):
            raise ValueError('Non supported field variation "{}".'.format(bai_field_variation))

        normalized_author_name = normalize_name(node_value).strip('.')

        if ElasticSearchVisitor.KEYWORD_TO_ES_FIELDNAME['author'] and \
                ElasticSearchVisitor.BAI_REGEX.match(node_value):
            return [ElasticSearchVisitor.AUTHORS_BAI_FIELD + '.' + bai_field_variation]

        elif not whitespace.search(normalized_author_name) and \
                query_bai_field_if_dots_in_name and \
                ElasticSearchVisitor.KEYWORD_TO_ES_FIELDNAME['author'] and \
                '.' in normalized_author_name:
            # Case of partial BAI, e.g. ``J.Smith``.
            return [ElasticSearchVisitor.AUTHORS_BAI_FIELD + '.' + bai_field_variation] + \
                   force_list(ElasticSearchVisitor.KEYWORD_TO_ES_FIELDNAME['author'])

        else:
            return None

Response: Generates new fieldnames in case of BAI query.


Instruction:
Implement a function in Python 3 to
generate a query handling specifically authors.

Response:
def _generate_author_query(self, author_name):
        """Generates a query handling specifically authors.

        Notes:
            The match query is generic enough to return many results. Then, using the filter clause we truncate these
            so that we imitate legacy's behaviour on returning more "exact" results. E.g. Searching for `Smith, John`
            shouldn't return papers of 'Smith, Bob'.

            Additionally, doing a ``match`` with ``"operator": "and"`` in order to be even more exact in our search, by
            requiring that ``full_name`` field contains both
        """
        name_variations = [name_variation.lower()
                           for name_variation
                           in generate_minimal_name_variations(author_name)]

        # When the query contains sufficient data, i.e. full names, e.g. ``Mele, Salvatore`` (and not ``Mele, S`` or
        # ``Mele``) we can improve our filtering in order to filter out results containing records with authors that
        # have the same non lastnames prefix, e.g. 'Mele, Samuele'.
        if author_name_contains_fullnames(author_name):
            specialized_author_filter = [
                {
                    'bool': {
                        'must': [
                            {
                                'term': {ElasticSearchVisitor.AUTHORS_NAME_VARIATIONS_FIELD: names_variation[0]}
                            },
                            generate_match_query(
                                ElasticSearchVisitor.KEYWORD_TO_ES_FIELDNAME['author'],
                                names_variation[1],
                                with_operator_and=True
                            )
                        ]
                    }
                } for names_variation
                in product(name_variations, name_variations)
            ]

        else:
            # In the case of initials or even single lastname search, filter with only the name variations.
            specialized_author_filter = [
                {'term': {ElasticSearchVisitor.AUTHORS_NAME_VARIATIONS_FIELD: name_variation}}
                for name_variation in name_variations
            ]

        query = {
            'bool': {
                'filter': {
                    'bool': {
                        'should': specialized_author_filter
                    }
                },
                'must': {
                    'match': {
                        ElasticSearchVisitor.KEYWORD_TO_ES_FIELDNAME['author']: author_name
                    }
                }
            }
        }

        return generate_nested_query(ElasticSearchVisitor.AUTHORS_NESTED_QUERY_PATH, query)


Instruction:
Create a Python 3 function to
generate a term query handling authors and BAIs.

Response:
def _generate_exact_author_query(self, author_name_or_bai):
        """Generates a term query handling authors and BAIs.

        Notes:
            If given value is a BAI, search for the provided value in the raw field variation of
            `ElasticSearchVisitor.AUTHORS_BAI_FIELD`.
            Otherwise, the value will be procesed in the same way as the indexed value (i.e. lowercased and normalized
            (inspire_utils.normalize_name and then NFKC normalization).
            E.g. Searching for 'Smith, J.' is the same as searching for: 'Smith, J', 'smith, j.', 'smith j', 'j smith',
            'j. smith', 'J Smith', 'J. Smith'.
        """
        if ElasticSearchVisitor.BAI_REGEX.match(author_name_or_bai):
            bai = author_name_or_bai.lower()
            query = self._generate_term_query(
                '.'.join((ElasticSearchVisitor.AUTHORS_BAI_FIELD, FieldVariations.search)),
                bai
            )
        else:
            author_name = normalize('NFKC', normalize_name(author_name_or_bai)).lower()
            query = self._generate_term_query(
                ElasticSearchVisitor.KEYWORD_TO_ES_FIELDNAME['exact-author'],
                author_name
            )

        return generate_nested_query(ElasticSearchVisitor.AUTHORS_NESTED_QUERY_PATH, query)


Instruction:
Here you have a function in Python 3, explain what it does
def _generate_queries_for_title_symbols(title_field, query_value):
        values_tokenized_by_whitespace = query_value.split()

        symbol_queries = []
        for value in values_tokenized_by_whitespace:
            # Heuristic: If there's a symbol-indicating-character in the value, it signifies terms that should be
            # queried against the whitespace-tokenized title.
            if any(character in value for character in ElasticSearchVisitor.TITLE_SYMBOL_INDICATING_CHARACTER):
                symbol_queries.append(
                    generate_match_query(
                        '.'.join([title_field, FieldVariations.search]),
                        value,
                        with_operator_and=False
                    )
                )

        return wrap_queries_in_bool_clauses_if_more_than_one(symbol_queries, use_must_clause=True)

Response: Generates a list of queries for any symbols in the title against the whitespace tokenized field of titles.


Instruction:
Can you generate the documentation for the following Python 3 function
def _generate_type_code_query(self, value):
        mapping_for_value = self.TYPECODE_VALUE_TO_FIELD_AND_VALUE_PAIRS_MAPPING.get(value, None)

        if mapping_for_value:
            return generate_match_query(*mapping_for_value, with_operator_and=True)
        else:
            return {
                'bool': {
                    'minimum_should_match': 1,
                    'should': [
                        generate_match_query('document_type', value, with_operator_and=True),
                        generate_match_query('publication_type', value, with_operator_and=True),
                    ]
                }
            }

Response: Generate type - code queries.


Instruction:
Can you generate the documentation for the following Python 3 function
def _generate_range_queries(self, fieldnames, operator_value_pairs):
        if ElasticSearchVisitor.KEYWORD_TO_ES_FIELDNAME['date'] == fieldnames:
            range_queries = []
            for fieldname in fieldnames:
                updated_operator_value_pairs = \
                    update_date_value_in_operator_value_pairs_for_fieldname(fieldname, operator_value_pairs)

                if not updated_operator_value_pairs:
                    break  # Malformed date
                else:
                    range_query = {
                        'range': {
                            fieldname: updated_operator_value_pairs
                        }
                    }

                    range_queries.append(
                        generate_nested_query(ElasticSearchVisitor.DATE_NESTED_QUERY_PATH, range_query)
                        if fieldname in ElasticSearchVisitor.DATE_NESTED_FIELDS
                        else range_query
                    )
        else:
            range_queries = [{
                    'range': {
                        fieldname: operator_value_pairs
                    }
                }
                for fieldname in fieldnames
            ]

        return wrap_queries_in_bool_clauses_if_more_than_one(range_queries, use_must_clause=False)

Response: Generates ElasticSearch range queries.


Instruction:
Can you implement a function in Python 3 that
generates a query string that can be used to search for the content of the _all field.

Response:
def _generate_malformed_query(data):
        """Generates a query on the ``_all`` field with all the query content.

        Args:
            data (six.text_type or list): The query in the format of ``six.text_type`` (when used from parsing driver)
                or ``list`` when used from withing the ES visitor.
        """
        if isinstance(data, six.text_type):
            # Remove colon character (special character for ES)
            query_str = data.replace(':', ' ')
        else:
            query_str = ' '.join([word.strip(':') for word in data.children])

        return {
            'simple_query_string': {
                'fields': ['_all'],
                'query': query_str
            }
        }


Instruction:
How would you explain what the following Python 3 function does
def _preprocess_journal_query_value(third_journal_field, old_publication_info_values):
        # Prepare old publication info for :meth:`inspire_schemas.utils.convert_old_publication_info_to_new`.
        publication_info_keys = [
            ElasticSearchVisitor.JOURNAL_TITLE,
            ElasticSearchVisitor.JOURNAL_VOLUME,
            third_journal_field,
        ]
        values_list = [
            value.strip()
            for value
            in old_publication_info_values.split(',')
            if value
        ]

        old_publication_info = [
            {
                key: value
                for key, value
                in zip(publication_info_keys, values_list)
                if value
            }
        ]

        # We are always assuming that the returned list will not be empty. In the situation of a journal query with no
        # value, a malformed query will be generated instead.
        new_publication_info = convert_old_publication_info_to_new(old_publication_info)[0]

        return new_publication_info

Response: Transforms the given journal query value to the new one.


Instruction:
How would you explain what the following Python 3 function does
def _generate_journal_nested_queries(self, value):
        # Abstract away which is the third field, we care only for its existence.
        third_journal_field = ElasticSearchVisitor.JOURNAL_PAGE_START

        new_publication_info = ElasticSearchVisitor._preprocess_journal_query_value(third_journal_field, value)

        # We always expect a journal title, otherwise query would be considered malformed, and thus this method would
        # not have been called.
        queries_for_each_field = [
            generate_match_query(ElasticSearchVisitor.JOURNAL_FIELDS_MAPPING[ElasticSearchVisitor.JOURNAL_TITLE],
                                 new_publication_info[ElasticSearchVisitor.JOURNAL_TITLE],
                                 with_operator_and=False)
        ]

        if ElasticSearchVisitor.JOURNAL_VOLUME in new_publication_info:
            queries_for_each_field.append(
                generate_match_query(
                    ElasticSearchVisitor.JOURNAL_FIELDS_MAPPING[ElasticSearchVisitor.JOURNAL_VOLUME],
                    new_publication_info[ElasticSearchVisitor.JOURNAL_VOLUME],
                    with_operator_and=False
                )
            )

        if third_journal_field in new_publication_info:
            artid_or_page_start = new_publication_info[third_journal_field]
            match_queries = [
                generate_match_query(
                    ElasticSearchVisitor.JOURNAL_FIELDS_MAPPING[third_field],
                    artid_or_page_start,
                    with_operator_and=False
                )
                for third_field
                in (ElasticSearchVisitor.JOURNAL_PAGE_START, ElasticSearchVisitor.JOURNAL_ART_ID)
            ]

            queries_for_each_field.append(
                wrap_queries_in_bool_clauses_if_more_than_one(match_queries, use_must_clause=False)
            )

        return generate_nested_query(
            ElasticSearchVisitor.JOURNAL_FIELDS_PREFIX,
            wrap_queries_in_bool_clauses_if_more_than_one(queries_for_each_field, use_must_clause=True)
        )

Response: Generates ElasticSearch nested queries for the given value.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def visit_exact_match_value(self, node, fieldnames=None):
        if not fieldnames:
            fieldnames = ['_all']
        else:
            fieldnames = force_list(fieldnames)

        if ElasticSearchVisitor.KEYWORD_TO_ES_FIELDNAME['exact-author'] == fieldnames[0]:
            return self._generate_exact_author_query(node.value)

        elif ElasticSearchVisitor.KEYWORD_TO_ES_FIELDNAME['type-code'] == fieldnames[0]:
            return self._generate_type_code_query(node.value)

        elif ElasticSearchVisitor.KEYWORD_TO_ES_FIELDNAME['journal'] == fieldnames:
            return self._generate_journal_nested_queries(node.value)

        bai_fieldnames = self._generate_fieldnames_if_bai_query(
            node.value,
            bai_field_variation=FieldVariations.raw,
            query_bai_field_if_dots_in_name=False
        )

        if ElasticSearchVisitor.KEYWORD_TO_ES_FIELDNAME['date'] == fieldnames:
            term_queries = []
            for field in fieldnames:
                term_query =  \
                    {'term': {field: _truncate_date_value_according_on_date_field(field, node.value).dumps()}}

                term_queries.append(
                    generate_nested_query(ElasticSearchVisitor.DATE_NESTED_QUERY_PATH, term_query)
                    if field in ElasticSearchVisitor.DATE_NESTED_FIELDS
                    else term_query
                )
        elif ElasticSearchVisitor.KEYWORD_TO_ES_FIELDNAME['author'] in fieldnames:
            term_queries = [
                generate_nested_query(ElasticSearchVisitor.AUTHORS_NESTED_QUERY_PATH, {'term': {field: node.value}})
                for field in (bai_fieldnames or fieldnames)
            ]
        else:
            term_queries = [{'term': {field: node.value}} for field in (bai_fieldnames or fieldnames)]

        return wrap_queries_in_bool_clauses_if_more_than_one(term_queries, use_must_clause=False)

Response: Generates a term query that matches the value of a node.


Instruction:
Can you tell what is the following Python 3 function doing
def visit_partial_match_value(self, node, fieldnames=None):
        if ElasticSearchVisitor.KEYWORD_TO_ES_FIELDNAME['date'] == fieldnames:
            # Date queries with partial values are transformed into range queries, among the given and the exact
            # next date, according to the granularity of the given date.
            if node.contains_wildcard:
                return self._generate_date_with_wildcard_query(node.value)

            return self._generate_range_queries(force_list(fieldnames), {ES_RANGE_EQ_OPERATOR: node.value})

        if ElasticSearchVisitor.KEYWORD_TO_ES_FIELDNAME['exact-author'] == fieldnames:
            return self._generate_exact_author_query(node.value)

        elif ElasticSearchVisitor.KEYWORD_TO_ES_FIELDNAME['type-code'] == fieldnames:
            return self._generate_type_code_query(node.value)

        elif ElasticSearchVisitor.KEYWORD_TO_ES_FIELDNAME['journal'] == fieldnames:
            return self._generate_journal_nested_queries(node.value)

        # Add wildcard token as prefix and suffix.
        value = \
            ('' if node.value.startswith(ast.GenericValue.WILDCARD_TOKEN) else '*') + \
            node.value + \
            ('' if node.value.endswith(ast.GenericValue.WILDCARD_TOKEN) else '*')

        bai_fieldnames = self._generate_fieldnames_if_bai_query(
            node.value,
            bai_field_variation=FieldVariations.search,
            query_bai_field_if_dots_in_name=True
        )

        query = self._generate_query_string_query(value,
                                                  fieldnames=bai_fieldnames or fieldnames,
                                                  analyze_wildcard=True)
        if (bai_fieldnames and ElasticSearchVisitor.KEYWORD_TO_ES_FIELDNAME['author'] in bai_fieldnames) \
                or (fieldnames and ElasticSearchVisitor.KEYWORD_TO_ES_FIELDNAME['author'] in fieldnames):
            return generate_nested_query(ElasticSearchVisitor.AUTHORS_NESTED_QUERY_PATH, query)

        return query

Response: Generates a query which looks for a substring of the node s value in the given fieldname.


Instruction:
Can you implement a function in Python 3 that
synchronises the schedule specified by the ID schedule_id to the scheduler service.

Response:
def run(self, schedule_id, **kwargs):
        """
        Synchronises the schedule specified by the ID `schedule_id` to the
        scheduler service.

        Arguments:
            schedule_id {str} -- The ID of the schedule to sync
        """
        log = self.get_logger(**kwargs)

        try:
            schedule = Schedule.objects.get(id=schedule_id)
        except Schedule.DoesNotExist:
            log.error("Missing Schedule %s", schedule_id, exc_info=True)

        if schedule.scheduler_schedule_id is None:
            # Create the new schedule
            result = self.scheduler.create_schedule(schedule.scheduler_format)
            schedule.scheduler_schedule_id = result["id"]
            # Disable update signal here to avoid calling twice
            post_save.disconnect(schedule_saved, sender=Schedule)
            schedule.save(update_fields=("scheduler_schedule_id",))
            post_save.connect(schedule_saved, sender=Schedule)

            log.info(
                "Created schedule %s on scheduler for schedule %s",
                schedule.scheduler_schedule_id,
                schedule.id,
            )
        else:
            # Update the existing schedule
            result = self.scheduler.update_schedule(
                str(schedule.scheduler_schedule_id), schedule.scheduler_format
            )
            log.info(
                "Updated schedule %s on scheduler for schedule %s",
                schedule.scheduler_schedule_id,
                schedule.id,
            )


Instruction:
Can you create a Python 3 function that
deactivates the schedule specified by the ID scheduler_schedule_id in the scheduler service.

Response:
def run(self, scheduler_schedule_id, **kwargs):
        """
        Deactivates the schedule specified by the ID `scheduler_schedule_id` in
        the scheduler service.

        Arguments:
            scheduler_schedule_id {str} -- The ID of the schedule to deactivate
        """
        log = self.get_logger(**kwargs)

        self.scheduler.update_schedule(scheduler_schedule_id, {"active": False})
        log.info(
            "Deactivated schedule %s in the scheduler service", scheduler_schedule_id
        )


Instruction:
Write a Python 3 script for
returning the tags for a specific model or content type.

Response:
def get_for_model(self, obj):
        """Returns the tags for a specific model/content type."""
        qs = Tag.objects.language(get_language())
        qs = qs.filter(
            tagged_items__content_type=ctype_models.ContentType.objects.get_for_model(obj))  # NOQA
        return qs.distinct()


Instruction:
Write a Python 3 script for
returning all tags for a whole queryset of objects.

Response:
def get_for_queryset(self, obj_queryset):
        """Returns all tags for a whole queryset of objects."""
        qs = Tag.objects.language(get_language())
        if obj_queryset.count() == 0:
            return qs.none()
        qs = qs.filter(
            tagged_items__object_id__in=[
                obj.id for obj in obj_queryset],
            tagged_items__content_type=ctype_models.ContentType.objects.get_for_model(obj_queryset[0]))  # NOQA
        return qs.distinct()


Instruction:
How would you code a function in Python 3 to
task to ensure subscription is bumped or converted

Response:
def post_send_process(context):
    """
    Task to ensure subscription is bumped or converted
    """
    if "error" in context:
        return context

    [deserialized_subscription] = serializers.deserialize(
        "json", context["subscription"]
    )
    subscription = deserialized_subscription.object
    [messageset] = serializers.deserialize("json", context["messageset"])
    messageset = messageset.object

    # Get set max
    set_max = messageset.messages.filter(lang=subscription.lang).count()
    logger.debug("set_max calculated - %s" % set_max)

    # Compare user position to max
    if subscription.next_sequence_number == set_max:
        with transaction.atomic():
            # Mark current as completed
            logger.debug("marking current subscription as complete")
            subscription.completed = True
            subscription.active = False
            subscription.process_status = 2  # Completed
            deserialized_subscription.save(
                update_fields=("completed", "active", "process_status")
            )
            # If next set defined create new subscription
            if messageset.next_set:
                logger.info("Creating new subscription for next set")
                newsub = Subscription.objects.create(
                    identity=subscription.identity,
                    lang=subscription.lang,
                    messageset=messageset.next_set,
                    schedule=messageset.next_set.default_schedule,
                )
                logger.debug("Created Subscription <%s>" % newsub.id)
    else:
        # More in this set so increment by one
        logger.debug("incrementing next_sequence_number")
        subscription.next_sequence_number = F("next_sequence_number") + 1
        logger.debug("setting process status back to 0")
        subscription.process_status = 0
        logger.debug("saving subscription")
        deserialized_subscription.save(
            update_fields=("next_sequence_number", "process_status")
        )
    # return response
    return "Subscription for %s updated" % str(subscription.id)


Instruction:
Can you generate the documentation for the following Python 3 function
def calculate_subscription_lifecycle(subscription_id):
    subscription = Subscription.objects.select_related("messageset", "schedule").get(
        id=subscription_id
    )
    behind = subscription.messages_behind()
    if behind == 0:
        return

    current_messageset = subscription.messageset
    current_sequence_number = subscription.next_sequence_number
    end_subscription = Subscription.fast_forward_lifecycle(subscription, save=False)[-1]
    BehindSubscription.objects.create(
        subscription=subscription,
        messages_behind=behind,
        current_messageset=current_messageset,
        current_sequence_number=current_sequence_number,
        expected_messageset=end_subscription.messageset,
        expected_sequence_number=end_subscription.next_sequence_number,
    )

Response:
Calculates the expected lifecycle position of the subscription in
   .


Instruction:
Given the following Python 3 function, write the documentation
def find_behind_subscriptions():
    subscriptions = Subscription.objects.filter(
        active=True, completed=False, process_status=0
    ).values_list("id", flat=True)
    for subscription_id in subscriptions.iterator():
        calculate_subscription_lifecycle.delay(str(subscription_id))

Response: Finds any subscriptions that are behind according to where they should be and creates a BehindSubscription entry for them.


Instruction:
Make a summary of the following Python 3 code
def send(self, request, pk=None):
        schedule = self.get_object()
        queue_subscription_send.delay(str(schedule.id))

        return Response({}, status=status.HTTP_202_ACCEPTED)

Response: Sends all the subscriptions for the specified schedule


Instruction:
Create a Python 3 function to
return the configured value for the given key name or default.

Response:
def get(self, key: Any, default: Any=None) -> Any:
        '''Return the configured value for the given key name, or `default` if
        no value is available or key is invalid.'''

        return self.data.get(key, default)


Instruction:
Can you write a function in Python 3 where it
returns the task - specific configuration.

Response:
def task_config(self, task: Task) -> Any:
        '''Return the task-specific configuration.'''

        return self.get(task.__class__.__name__)


Instruction:
Create a Python 3 function for
loading configuration in JSON format from either a file or a raw data string.

Response:
async def init(self) -> None:
        '''Load configuration in JSON format from either a file or
        a raw data string.'''

        if self.data:
            return

        if self.json_data:
            try:
                self.data = json.loads(self.json_data)

            except Exception:
                Log.exception('Falied to load raw configuration')

        else:
            try:
                with open(self.json_path, 'r') as f:
                    self.data = json.load(f)

            except Exception:
                Log.exception('Failed to load configuration from %s',
                              self.json_path)
                self.data = {}


Instruction:
Write a Python 3 function for
executing the task inside the asyncio event loop after self. DELAY seconds.

Response:
async def run_task(self) -> None:
        '''Execute the task inside the asyncio event loop after `DELAY`
        seconds.  Track the time it takes to run, and log when it starts/stops.
        If/when `reset()` is called, reset the wait time to `DELAY` seconds.'''

        self.last_run = 0.0
        self.target = self.time() + self.DELAY

        while self.running:
            try:
                now = self.time()

                if now < self.target:
                    sleep = self.target - now
                    await self.sleep(sleep)

                elif self.last_run < self.target:
                    Log.debug('executing timer task %s', self.name)
                    self.last_run = self.time()
                    await self.run()
                    total = self.time() - self.last_run
                    Log.debug('finished timer task %s in %.1f seconds',
                              self.name, total)

                else:
                    sleep = min(5.0, self.DELAY)
                    await self.sleep(sleep)

            except CancelledError:
                Log.debug('cancelled timer task %s', self.name)
                raise

            except Exception:
                Log.exception('exception in timer task %s', self.name)


Instruction:
Explain what the following Python 3 code does
def reset(self) -> None:
        '''Reset task execution to `DELAY` seconds from now.'''

        Log.debug('resetting timer task %s')
        self.target = self.time() + self.DELAY

Response: Reset the task execution to the current time.


Instruction:
Explain what the following Python 3 code does
def internal_only(view_func):

    @functools.wraps(view_func)
    def wrapper(request, *args, **kwargs):
        forwards = request.META.get("HTTP_X_FORWARDED_FOR", "").split(",")
        # The nginx in the docker container adds the loadbalancer IP to the list inside
        # X-Forwarded-For, so if the list contains more than a single item, we know
        # that it went through our loadbalancer
        if len(forwards) > 1:
            raise PermissionDenied()
        return view_func(request, *args, **kwargs)

    return wrapper

Response: A view decorator which blocks access for requests coming through the load balancer.


Instruction:
Can you implement a function in Python 3 that
validates subscription data before creating Outbound message

Response:
def post(self, request, *args, **kwargs):
        """ Validates subscription data before creating Outbound message
        """
        schedule_disable.delay(kwargs["subscription_id"])
        return Response({"accepted": True}, status=201)


Instruction:
Given the following Python 3 function, write the documentation
def post(self, request, *args, **kwargs):
        # Look up subscriber
        subscription_id = kwargs["subscription_id"]
        if Subscription.objects.filter(id=subscription_id).exists():
            status = 202
            accepted = {"accepted": True}
            store_resend_request.apply_async(args=[subscription_id])
        else:
            status = 400
            accepted = {
                "accepted": False,
                "reason": "Cannot find subscription with ID {}".format(subscription_id),
            }
        return Response(accepted, status=status)

Response: Validates subscription data before creating Outbound message


Instruction:
Write a Python 3 function for
validating subscription data before creating Subscription message

Response:
def post(self, request, *args, **kwargs):
        """ Validates subscription data before creating Subscription message
        """
        # Ensure that we check for the 'data' key in the request object before
        # attempting to reference it
        if "data" in request.data:
            # This is a workaround for JSONField not liking blank/null refs
            if "metadata" not in request.data["data"]:
                request.data["data"]["metadata"] = {}

            if "initial_sequence_number" not in request.data["data"]:
                request.data["data"]["initial_sequence_number"] = request.data[
                    "data"
                ].get("next_sequence_number")

            subscription = SubscriptionSerializer(data=request.data["data"])
            if subscription.is_valid():
                subscription.save()
                # Return
                status = 201
                accepted = {"accepted": True}
                return Response(accepted, status=status)
            else:
                status = 400
                return Response(subscription.errors, status=status)
        else:
            status = 400
            message = {"data": ["This field is required."]}
            return Response(message, status=status)


Instruction:
Explain what the following Python 3 code does
def find_behind_subscriptions(self, request):

        task_id = find_behind_subscriptions.delay()
        return Response(
            {"accepted": True, "task_id": str(task_id)}, status=status.HTTP_202_ACCEPTED
        )

Response:
Starts a celery task that looks through active subscriptions to find
        and adds a
            BehindSubscription for them.


Instruction:
Here you have a function in Python 3, explain what it does
def repl():
    while True:
        try:
            sys.stdout.write("Type in next query: \n> ")
            import locale
            query_str = raw_input().decode(sys.stdin.encoding or locale.getpreferredencoding(True))
        except KeyboardInterrupt:
            break

        if u'quit' in query_str:
            break

        print_query_and_parse_tree(query_str)

Response: Read - Eval - Print - Loop for reading the query printing it and its parse tree.