Instruction:
How would you code a function in Python 3 to
get attribute by name.

Response:
def get_attribute_by_name(el, name, default=None):
        """Get attribute by name."""

        value = default
        if el._is_xml:
            try:
                value = el.attrs[name]
            except KeyError:
                pass
        else:
            for k, v in el.attrs.items():
                if util.lower(k) == name:
                    value = v
                    break
        return value


Instruction:
Explain what the following Python 3 code does
def get_text(self, el, no_iframe=False):

        return ''.join(
            [node for node in self.get_descendants(el, tags=False, no_iframe=no_iframe) if self.is_content_string(node)]
        )

Response: Get text from the element.


Instruction:
How would you code a function in Python 3 to
parse the input value.

Response:
def parse_value(cls, itype, value):
        """Parse the input value."""

        parsed = None
        if itype == "date":
            m = RE_DATE.match(value)
            if m:
                year = int(m.group('year'), 10)
                month = int(m.group('month'), 10)
                day = int(m.group('day'), 10)
                if cls.validate_year(year) and cls.validate_month(month) and cls.validate_day(year, month, day):
                    parsed = (year, month, day)
        elif itype == "month":
            m = RE_MONTH.match(value)
            if m:
                year = int(m.group('year'), 10)
                month = int(m.group('month'), 10)
                if cls.validate_year(year) and cls.validate_month(month):
                    parsed = (year, month)
        elif itype == "week":
            m = RE_WEEK.match(value)
            if m:
                year = int(m.group('year'), 10)
                week = int(m.group('week'), 10)
                if cls.validate_year(year) and cls.validate_week(year, week):
                    parsed = (year, week)
        elif itype == "time":
            m = RE_TIME.match(value)
            if m:
                hour = int(m.group('hour'), 10)
                minutes = int(m.group('minutes'), 10)
                if cls.validate_hour(hour) and cls.validate_minutes(minutes):
                    parsed = (hour, minutes)
        elif itype == "datetime-local":
            m = RE_DATETIME.match(value)
            if m:
                year = int(m.group('year'), 10)
                month = int(m.group('month'), 10)
                day = int(m.group('day'), 10)
                hour = int(m.group('hour'), 10)
                minutes = int(m.group('minutes'), 10)
                if (
                    cls.validate_year(year) and cls.validate_month(month) and cls.validate_day(year, month, day) and
                    cls.validate_hour(hour) and cls.validate_minutes(minutes)
                ):
                    parsed = (year, month, day, hour, minutes)
        elif itype in ("number", "range"):
            m = RE_NUM.match(value)
            if m:
                parsed = float(m.group('value'))
        return parsed


Instruction:
Can you tell what is the following Python 3 function doing
def find_bidi(self, el):

        for node in self.get_children(el, tags=False):

            # Analyze child text nodes
            if self.is_tag(node):

                # Avoid analyzing certain elements specified in the specification.
                direction = DIR_MAP.get(util.lower(self.get_attribute_by_name(node, 'dir', '')), None)
                if (
                    self.get_tag(node) in ('bdi', 'script', 'style', 'textarea', 'iframe') or
                    not self.is_html_tag(node) or
                    direction is not None
                ):
                    continue  # pragma: no cover

                # Check directionality of this node's text
                value = self.find_bidi(node)
                if value is not None:
                    return value

                # Direction could not be determined
                continue  # pragma: no cover

            # Skip `doctype` comments, etc.
            if self.is_special_string(node):
                continue

            # Analyze text nodes for directionality.
            for c in node:
                bidi = unicodedata.bidirectional(c)
                if bidi in ('AL', 'R', 'L'):
                    return ct.SEL_DIR_LTR if bidi == 'L' else ct.SEL_DIR_RTL
        return None

Response: Find the bidi of an element.


Instruction:
Create a Python 3 function for
matching attribute name and return value if it exists.

Response:
def match_attribute_name(self, el, attr, prefix):
        """Match attribute name and return value if it exists."""

        value = None
        if self.supports_namespaces():
            value = None
            # If we have not defined namespaces, we can't very well find them, so don't bother trying.
            if prefix:
                ns = self.namespaces.get(prefix)
                if ns is None and prefix != '*':
                    return None
            else:
                ns = None

            for k, v in self.iter_attributes(el):

                # Get attribute parts
                namespace, name = self.split_namespace(el, k)

                # Can't match a prefix attribute as we haven't specified one to match
                # Try to match it normally as a whole `p:a` as selector may be trying `p\:a`.
                if ns is None:
                    if (self.is_xml and attr == k) or (not self.is_xml and util.lower(attr) == util.lower(k)):
                        value = v
                        break
                    # Coverage is not finding this even though it is executed.
                    # Adding a print statement before this (and erasing coverage) causes coverage to find the line.
                    # Ignore the false positive message.
                    continue  # pragma: no cover

                # We can't match our desired prefix attribute as the attribute doesn't have a prefix
                if namespace is None or ns != namespace and prefix != '*':
                    continue

                # The attribute doesn't match.
                if (util.lower(attr) != util.lower(name)) if not self.is_xml else (attr != name):
                    continue

                value = v
                break
        else:
            for k, v in self.iter_attributes(el):
                if util.lower(attr) != util.lower(k):
                    continue
                value = v
                break
        return value


Instruction:
Here you have a function in Python 3, explain what it does
def match_namespace(self, el, tag):

        match = True
        namespace = self.get_tag_ns(el)
        default_namespace = self.namespaces.get('')
        tag_ns = '' if tag.prefix is None else self.namespaces.get(tag.prefix, None)
        # We must match the default namespace if one is not provided
        if tag.prefix is None and (default_namespace is not None and namespace != default_namespace):
            match = False
        # If we specified `|tag`, we must not have a namespace.
        elif (tag.prefix is not None and tag.prefix == '' and namespace):
            match = False
        # Verify prefix matches
        elif (
            tag.prefix and
            tag.prefix != '*' and (tag_ns is None or namespace != tag_ns)
        ):
            match = False
        return match

Response: Match the namespace of the element.


Instruction:
Create a Python 3 function to
match relationship to other elements.

Response:
def match_relations(self, el, relation):
        """Match relationship to other elements."""

        found = False

        if relation[0].rel_type.startswith(':'):
            found = self.match_future_relations(el, relation)
        else:
            found = self.match_past_relations(el, relation)

        return found


Instruction:
Given the following Python 3 function, write the documentation
def match_id(self, el, ids):

        found = True
        for i in ids:
            if i != self.get_attribute_by_name(el, 'id', ''):
                found = False
                break
        return found

Response: Match element s ID.


Instruction:
Make a summary of the following Python 3 code
def match_classes(self, el, classes):

        current_classes = self.get_classes(el)
        found = True
        for c in classes:
            if c not in current_classes:
                found = False
                break
        return found

Response: Match element s classes.


Instruction:
Here you have a function in Python 3, explain what it does
def match_nth_tag_type(self, el, child):

        return(
            (self.get_tag(child) == self.get_tag(el)) and
            (self.get_tag_ns(child) == self.get_tag_ns(el))
        )

Response: Match tag type for nth matches.


Instruction:
Given the following Python 3 function, write the documentation
def match_empty(self, el):

        is_empty = True
        for child in self.get_children(el, tags=False):
            if self.is_tag(child):
                is_empty = False
                break
            elif self.is_content_string(child) and RE_NOT_EMPTY.search(child):
                is_empty = False
                break
        return is_empty

Response: Check if element is empty.


Instruction:
Can you generate the documentation for the following Python 3 function
def match_contains(self, el, contains):

        match = True
        content = None
        for contain_list in contains:
            if content is None:
                content = self.get_text(el, no_iframe=self.is_html)
            found = False
            for text in contain_list.text:
                if text in content:
                    found = True
                    break
            if not found:
                match = False
        return match

Response: Match element if it contains text.


Instruction:
Write a Python 3 function for
matching the input element with the default form.

Response:
def match_default(self, el):
        """Match default."""

        match = False

        # Find this input's form
        form = None
        parent = self.get_parent(el, no_iframe=True)
        while parent and form is None:
            if self.get_tag(parent) == 'form' and self.is_html_tag(parent):
                form = parent
            else:
                parent = self.get_parent(parent, no_iframe=True)

        # Look in form cache to see if we've already located its default button
        found_form = False
        for f, t in self.cached_default_forms:
            if f is form:
                found_form = True
                if t is el:
                    match = True
                break

        # We didn't have the form cached, so look for its default button
        if not found_form:
            for child in self.get_descendants(form, no_iframe=True):
                name = self.get_tag(child)
                # Can't do nested forms (haven't figured out why we never hit this)
                if name == 'form':  # pragma: no cover
                    break
                if name in ('input', 'button'):
                    v = self.get_attribute_by_name(child, 'type', '')
                    if v and util.lower(v) == 'submit':
                        self.cached_default_forms.append([form, child])
                        if el is child:
                            match = True
                        break
        return match


Instruction:
How would you explain what the following Python 3 function does
def match_indeterminate(self, el):

        match = False
        name = self.get_attribute_by_name(el, 'name')

        def get_parent_form(el):
            """Find this input's form."""
            form = None
            parent = self.get_parent(el, no_iframe=True)
            while form is None:
                if self.get_tag(parent) == 'form' and self.is_html_tag(parent):
                    form = parent
                    break
                last_parent = parent
                parent = self.get_parent(parent, no_iframe=True)
                if parent is None:
                    form = last_parent
                    break
            return form

        form = get_parent_form(el)

        # Look in form cache to see if we've already evaluated that its fellow radio buttons are indeterminate
        found_form = False
        for f, n, i in self.cached_indeterminate_forms:
            if f is form and n == name:
                found_form = True
                if i is True:
                    match = True
                break

        # We didn't have the form cached, so validate that the radio button is indeterminate
        if not found_form:
            checked = False
            for child in self.get_descendants(form, no_iframe=True):
                if child is el:
                    continue
                tag_name = self.get_tag(child)
                if tag_name == 'input':
                    is_radio = False
                    check = False
                    has_name = False
                    for k, v in self.iter_attributes(child):
                        if util.lower(k) == 'type' and util.lower(v) == 'radio':
                            is_radio = True
                        elif util.lower(k) == 'name' and v == name:
                            has_name = True
                        elif util.lower(k) == 'checked':
                            check = True
                        if is_radio and check and has_name and get_parent_form(child) is form:
                            checked = True
                            break
                if checked:
                    break
            if not checked:
                match = True
            self.cached_indeterminate_forms.append([form, name, match])

        return match

Response: Match an indeterminate form.


Instruction:
Can you generate the documentation for the following Python 3 function
def match_dir(self, el, directionality):

        # If we have to match both left and right, we can't match either.
        if directionality & ct.SEL_DIR_LTR and directionality & ct.SEL_DIR_RTL:
            return False

        if el is None or not self.is_html_tag(el):
            return False

        # Element has defined direction of left to right or right to left
        direction = DIR_MAP.get(util.lower(self.get_attribute_by_name(el, 'dir', '')), None)
        if direction not in (None, 0):
            return direction == directionality

        # Element is the document element (the root) and no direction assigned, assume left to right.
        is_root = self.is_root(el)
        if is_root and direction is None:
            return ct.SEL_DIR_LTR == directionality

        # If `input[type=telephone]` and no direction is assigned, assume left to right.
        name = self.get_tag(el)
        is_input = name == 'input'
        is_textarea = name == 'textarea'
        is_bdi = name == 'bdi'
        itype = util.lower(self.get_attribute_by_name(el, 'type', '')) if is_input else ''
        if is_input and itype == 'tel' and direction is None:
            return ct.SEL_DIR_LTR == directionality

        # Auto handling for text inputs
        if ((is_input and itype in ('text', 'search', 'tel', 'url', 'email')) or is_textarea) and direction == 0:
            if is_textarea:
                value = []
                for node in self.get_contents(el, no_iframe=True):
                    if self.is_content_string(node):
                        value.append(node)
                value = ''.join(value)
            else:
                value = self.get_attribute_by_name(el, 'value', '')
            if value:
                for c in value:
                    bidi = unicodedata.bidirectional(c)
                    if bidi in ('AL', 'R', 'L'):
                        direction = ct.SEL_DIR_LTR if bidi == 'L' else ct.SEL_DIR_RTL
                        return direction == directionality
                # Assume left to right
                return ct.SEL_DIR_LTR == directionality
            elif is_root:
                return ct.SEL_DIR_LTR == directionality
            return self.match_dir(self.get_parent(el, no_iframe=True), directionality)

        # Auto handling for `bdi` and other non text inputs.
        if (is_bdi and direction is None) or direction == 0:
            direction = self.find_bidi(el)
            if direction is not None:
                return direction == directionality
            elif is_root:
                return ct.SEL_DIR_LTR == directionality
            return self.match_dir(self.get_parent(el, no_iframe=True), directionality)

        # Match parents direction
        return self.match_dir(self.get_parent(el, no_iframe=True), directionality)

Response: Check if the element is in the given directionality.


Instruction:
How would you explain what the following Python 3 function does
def match_range(self, el, condition):

        out_of_range = False

        itype = self.get_attribute_by_name(el, 'type').lower()
        mn = self.get_attribute_by_name(el, 'min', None)
        if mn is not None:
            mn = Inputs.parse_value(itype, mn)
        mx = self.get_attribute_by_name(el, 'max', None)
        if mx is not None:
            mx = Inputs.parse_value(itype, mx)

        # There is no valid min or max, so we cannot evaluate a range
        if mn is None and mx is None:
            return False

        value = self.get_attribute_by_name(el, 'value', None)
        if value is not None:
            value = Inputs.parse_value(itype, value)
        if value is not None:
            if itype in ("date", "datetime-local", "month", "week", "number", "range"):
                if mn is not None and value < mn:
                    out_of_range = True
                if not out_of_range and mx is not None and value > mx:
                    out_of_range = True
            elif itype == "time":
                if mn is not None and mx is not None and mn > mx:
                    # Time is periodic, so this is a reversed/discontinuous range
                    if value < mn and value > mx:
                        out_of_range = True
                else:
                    if mn is not None and value < mn:
                        out_of_range = True
                    if not out_of_range and mx is not None and value > mx:
                        out_of_range = True

        return not out_of_range if condition & ct.SEL_IN_RANGE else out_of_range

Response: Match the range of the elements in the element el.


Instruction:
Can you tell what is the following Python 3 function doing
def match_defined(self, el):

        name = self.get_tag(el)
        return (
            name.find('-') == -1 or
            name.find(':') != -1 or
            self.get_prefix(el) is not None
        )

Response:
Match defined.

        `:defined` is related to custom elements in a browser.

        - If the document is XML (not XHTML), all tags will match.
        - Tags that are not custom (don't have a hyphen) are marked defined.
        - If the tag has a prefix (without or without a namespace), it will not match.

        This is of course requires the parser to provide us with the proper prefix and namespace info,
        if it doesn't, there is nothing we can do.


Instruction:
Here you have a function in Python 3, explain what it does
def match_selectors(self, el, selectors):

        match = False
        is_not = selectors.is_not
        is_html = selectors.is_html

        # Internal selector lists that use the HTML flag, will automatically get the `html` namespace.
        if is_html:
            namespaces = self.namespaces
            iframe_restrict = self.iframe_restrict
            self.namespaces = {'html': NS_XHTML}
            self.iframe_restrict = True

        if not is_html or self.is_html:
            for selector in selectors:
                match = is_not
                # We have a un-matchable situation (like `:focus` as you can focus an element in this environment)
                if isinstance(selector, ct.SelectorNull):
                    continue
                # Verify tag matches
                if not self.match_tag(el, selector.tag):
                    continue
                # Verify tag is defined
                if selector.flags & ct.SEL_DEFINED and not self.match_defined(el):
                    continue
                # Verify element is root
                if selector.flags & ct.SEL_ROOT and not self.match_root(el):
                    continue
                # Verify element is scope
                if selector.flags & ct.SEL_SCOPE and not self.match_scope(el):
                    continue
                # Verify `nth` matches
                if not self.match_nth(el, selector.nth):
                    continue
                if selector.flags & ct.SEL_EMPTY and not self.match_empty(el):
                    continue
                # Verify id matches
                if selector.ids and not self.match_id(el, selector.ids):
                    continue
                # Verify classes match
                if selector.classes and not self.match_classes(el, selector.classes):
                    continue
                # Verify attribute(s) match
                if not self.match_attributes(el, selector.attributes):
                    continue
                # Verify ranges
                if selector.flags & RANGES and not self.match_range(el, selector.flags & RANGES):
                    continue
                # Verify language patterns
                if selector.lang and not self.match_lang(el, selector.lang):
                    continue
                # Verify pseudo selector patterns
                if selector.selectors and not self.match_subselectors(el, selector.selectors):
                    continue
                # Verify relationship selectors
                if selector.relation and not self.match_relations(el, selector.relation):
                    continue
                # Validate that the current default selector match corresponds to the first submit button in the form
                if selector.flags & ct.SEL_DEFAULT and not self.match_default(el):
                    continue
                # Validate that the unset radio button is among radio buttons with the same name in a form that are
                # also not set.
                if selector.flags & ct.SEL_INDETERMINATE and not self.match_indeterminate(el):
                    continue
                # Validate element directionality
                if selector.flags & DIR_FLAGS and not self.match_dir(el, selector.flags & DIR_FLAGS):
                    continue
                # Validate that the tag contains the specified text.
                if not self.match_contains(el, selector.contains):
                    continue
                match = not is_not
                break

        # Restore actual namespaces being used for external selector lists
        if is_html:
            self.namespaces = namespaces
            self.iframe_restrict = iframe_restrict

        return match

Response: Check if element matches one of the selectors.


Instruction:
Create a Python 3 function to
match all tags under the targeted tag.

Response:
def select(self, limit=0):
        """Match all tags under the targeted tag."""

        if limit < 1:
            limit = None

        for child in self.get_descendants(self.tag):
            if self.match(child):
                yield child
                if limit is not None:
                    limit -= 1
                    if limit < 1:
                        break


Instruction:
Can you generate the documentation for the following Python 3 function
def filter(self):  # noqa A001

        return [tag for tag in self.get_contents(self.tag) if not self.is_navigable_string(tag) and self.match(tag)]

Response: Filter tag s children.


Instruction:
Given the following Python 3 function, write the documentation
def match(self, tag):

        return CSSMatch(self.selectors, tag, self.namespaces, self.flags).match(tag)

Response: Match the given tag.


Instruction:
Can you generate the documentation for the following Python 3 function
def closest(self, tag):

        return CSSMatch(self.selectors, tag, self.namespaces, self.flags).closest()

Response: Return the closest ancestor.


Instruction:
Implement a Python 3 function for
filtering an iterable of nodes.

Response:
def filter(self, iterable):  # noqa A001
        """
        Filter.

        `CSSMatch` can cache certain searches for tags of the same document,
        so if we are given a tag, all tags are from the same document,
        and we can take advantage of the optimization.

        Any other kind of iterable could have tags from different documents or detached tags,
        so for those, we use a new `CSSMatch` for each item in the iterable.
        """

        if CSSMatch.is_tag(iterable):
            return CSSMatch(self.selectors, iterable, self.namespaces, self.flags).filter()
        else:
            return [node for node in iterable if not CSSMatch.is_navigable_string(node) and self.match(node)]


Instruction:
Create a Python 3 function to
select a single tag.

Response:
def select_one(self, tag):
        """Select a single tag."""

        tags = self.select(tag, limit=1)
        return tags[0] if tags else None


Instruction:
Write a Python 3 function that can
iterate the specified tags.

Response:
def iselect(self, tag, limit=0):
        """Iterate the specified tags."""

        for el in CSSMatch(self.selectors, tag, self.namespaces, self.flags).select(limit):
            yield el


Instruction:
Make a summary of the following Python 3 code
def deprecated(message, stacklevel=2):  # pragma: no cover

    def _decorator(func):
        @wraps(func)
        def _func(*args, **kwargs):
            warnings.warn(
                "'{}' is deprecated. {}".format(func.__name__, message),
                category=DeprecationWarning,
                stacklevel=stacklevel
            )
            return func(*args, **kwargs)
        return _func
    return _decorator

Response: A decorator that marks a function as deprecated.


Instruction:
Can you create a Python 3 function that
gets the pattern context.

Response:
def get_pattern_context(pattern, index):
    """Get the pattern context."""

    last = 0
    current_line = 1
    col = 1
    text = []
    line = 1

    # Split pattern by newline and handle the text before the newline
    for m in RE_PATTERN_LINE_SPLIT.finditer(pattern):
        linetext = pattern[last:m.start(0)]
        if not len(m.group(0)) and not len(text):
            indent = ''
            offset = -1
            col = index - last + 1
        elif last <= index < m.end(0):
            indent = '--> '
            offset = (-1 if index > m.start(0) else 0) + 3
            col = index - last + 1
        else:
            indent = '    '
            offset = None
        if len(text):
            # Regardless of whether we are presented with `\r\n`, `\r`, or `\n`,
            # we will render the output with just `\n`. We will still log the column
            # correctly though.
            text.append('\n')
        text.append('{}{}'.format(indent, linetext))
        if offset is not None:
            text.append('\n')
            text.append(' ' * (col + offset) + '^')
            line = current_line

        current_line += 1
        last = m.end(0)

    return ''.join(text), line, col


Instruction:
Implement a function in Python 3 to
filter list of nodes.

Response:
def filter(select, iterable, namespaces=None, flags=0, **kwargs):  # noqa: A001
    """Filter list of nodes."""

    return compile(select, namespaces, flags, **kwargs).filter(iterable)


Instruction:
How would you explain what the following Python 3 function does
def select_one(select, tag, namespaces=None, flags=0, **kwargs):

    return compile(select, namespaces, flags, **kwargs).select_one(tag)

Response: Select a single tag.


Instruction:
Can you write a function in Python 3 where it
selects the specified tags.

Response:
def select(select, tag, namespaces=None, limit=0, flags=0, **kwargs):
    """Select the specified tags."""

    return compile(select, namespaces, flags, **kwargs).select(tag, limit)


Instruction:
Can you generate the documentation for the following Python 3 function
def iselect(select, tag, namespaces=None, limit=0, flags=0, **kwargs):

    for el in compile(select, namespaces, flags, **kwargs).iselect(tag, limit):
        yield el

Response: Iterate the specified tags.


Instruction:
Given the following Python 3 function, write the documentation
def all(cls, connection=None, **params):

        request = cls._make_request('GET', cls._get_all_path(), connection, params=params)
        return cls._create_object(request, connection=connection)

Response: Returns a list of all the user s infrastructure.


Instruction:
How would you explain what the following Python 3 function does
def iterall(cls, connection=None, **kwargs):

        try:
            limit = kwargs['limit']
        except KeyError:
            limit = None

        try:
            page = kwargs['page']
        except KeyError:
            page = None

        def _all_responses():
            page = 1  # one based
            params = kwargs.copy()

            while True:
                params.update(page=page, limit=250)
                rsp = cls._make_request('GET', cls._get_all_path(), connection, params=params)
                if rsp:
                    yield rsp
                    page += 1
                else:
                    yield []  # needed for case where there is no objects
                    break

        if not (limit or page):
            for rsp in _all_responses():
                for obj in rsp:
                    yield cls._create_object(obj, connection=connection)

        else:
            response = cls._make_request('GET', cls._get_all_path(), connection, params=kwargs)
            for obj in cls._create_object(response, connection=connection):
                yield obj

Response: Returns an autopaging generator that yields each object returned one by one.


Instruction:
Can you tell what is the following Python 3 function doing
def get(self, resource="", rid=None, **query):
        if rid:
            if resource[-1] != '/':
                resource += '/'
            resource += str(rid)
        response = self._run_method('GET', resource, query=query)
        return self._handle_response(resource, response)

Response: Retrieves the resource with given id rid or all resources with given id rid or all resources with given id rid.


Instruction:
Can you create a Python 3 function that
updates the resource with id rid with the given updates dictionary.

Response:
def update(self, resource, rid, updates):
        """
        Updates the resource with id 'rid' with the given updates dictionary.
        """
        if resource[-1] != '/':
            resource += '/'
        resource += str(rid)
        return self.put(resource, data=updates)


Instruction:
Here you have a function in Python 3, explain what it does
def delete(self, resource, rid=None):  # note that rid can't be 0 - problem?
        if rid:
            if resource[-1] != '/':
                resource += '/'
            resource += str(rid)
        response = self._run_method('DELETE', resource)
        return self._handle_response(resource, response, suppress_empty=True)

Response: Delete the resource with given id rid.


Instruction:
Can you tell what is the following Python 3 function doing
def put(self, url, data):
        response = self._run_method('PUT', url, data=data)
        log.debug("OUTPUT: %s" % response.content)
        return self._handle_response(url, response)

Response: Make a PUT request to save data.


Instruction:
Can you generate the documentation for the following Python 3 function
def post(self, url, data, headers={}):
        response = self._run_method('POST', url, data=data, headers=headers)
        return self._handle_response(url, response)

Response: POST request for creating new objects.


Instruction:
Implement a function in Python 3 to
handle the response from the server and returns the parsed JSON or raises an exception appropriately.

Response:
def _handle_response(self, url, res, suppress_empty=True):
        """
        Returns parsed JSON or raises an exception appropriately.
        """
        self._last_response = res
        result = {}
        if res.status_code in (200, 201, 202):
            try:
                result = res.json()
            except Exception as e:  # json might be invalid, or store might be down
                e.message += " (_handle_response failed to decode JSON: " + str(res.content) + ")"
                raise  # TODO better exception
        elif res.status_code == 204 and not suppress_empty:
            raise EmptyResponseWarning("%d %s @ %s: %s" % (res.status_code, res.reason, url, res.content), res)
        elif res.status_code >= 500:
            raise ServerException("%d %s @ %s: %s" % (res.status_code, res.reason, url, res.content), res)
        elif res.status_code == 429:
            raise RateLimitingException("%d %s @ %s: %s" % (res.status_code, res.reason, url, res.content), res)
        elif res.status_code >= 400:
            raise ClientRequestException("%d %s @ %s: %s" % (res.status_code, res.reason, url, res.content), res)
        elif res.status_code >= 300:
            raise RedirectionException("%d %s @ %s: %s" % (res.status_code, res.reason, url, res.content), res)
        return result


Instruction:
How would you implement a function in Python 3 that
verifies that a signed payload is valid and return the user s data.

Response:
def verify_payload(signed_payload, client_secret):
        """
        Given a signed payload (usually passed as parameter in a GET request to the app's load URL) and a client secret,
        authenticates the payload and returns the user's data, or False on fail.

        Uses constant-time str comparison to prevent vulnerability to timing attacks.
        """
        encoded_json, encoded_hmac = signed_payload.split('.')
        dc_json = base64.b64decode(encoded_json)
        signature = base64.b64decode(encoded_hmac)
        expected_sig = hmac.new(client_secret.encode(), base64.b64decode(encoded_json), hashlib.sha256).hexdigest()
        authorised = hmac.compare_digest(signature, expected_sig.encode())
        return json.loads(dc_json.decode()) if authorised else False


Instruction:
Create a Python 3 function to
fetch a token from given token_url and sets session headers for the session.

Response:
def fetch_token(self, client_secret, code, context, scope, redirect_uri,
                    token_url='https://login.bigcommerce.com/oauth2/token'):
        """
        Fetches a token from given token_url, using given parameters, and sets up session headers for
        future requests.
        redirect_uri should be the same as your callback URL.
        code, context, and scope should be passed as parameters to your callback URL on app installation.

        Raises HttpException on failure (same as Connection methods).
        """
        res = self.post(token_url, {'client_id': self.client_id,
                                    'client_secret': client_secret,
                                    'code': code,
                                    'context': context,
                                    'scope': scope,
                                    'grant_type': 'authorization_code',
                                    'redirect_uri': redirect_uri},
                        headers={'Content-Type': 'application/x-www-form-urlencoded'})
        self._session.headers.update(self._oauth_headers(self.client_id, res['access_token']))
        return res


Instruction:
Can you generate the documentation for the following Python 3 function
def _handle_response(self, url, res, suppress_empty=True):
        result = Connection._handle_response(self, url, res, suppress_empty)
        if 'X-Rate-Limit-Time-Reset-Ms' in res.headers:
            self.rate_limit = dict(ms_until_reset=int(res.headers['X-Rate-Limit-Time-Reset-Ms']),
                                   window_size_ms=int(res.headers['X-Rate-Limit-Time-Window-Ms']),
                                   requests_remaining=int(res.headers['X-Rate-Limit-Requests-Left']),
                                   requests_quota=int(res.headers['X-Rate-Limit-Requests-Quota']))
            if self.rate_limiting_management:
                if self.rate_limiting_management['min_requests_remaining'] >= self.rate_limit['requests_remaining']:
                    if self.rate_limiting_management['wait']:
                        sleep(ceil(float(self.rate_limit['ms_until_reset']) / 1000))
                    if self.rate_limiting_management.get('callback_function'):
                        callback = self.rate_limiting_management['callback_function']
                        args_dict = self.rate_limiting_management.get('callback_args')
                        if args_dict:
                            callback(args_dict)
                        else:
                            callback()

        return result

Response: Handles the response from the server.


Instruction:
Can you implement a function in Python 3 that
escapes a string of HTML.

Response:
def escape_html(text, escape_slash=False):
    """
    Binding for Hoedown's HTML escaping function.

    The implementation is inspired by the OWASP XSS Prevention recommendations:

    .. code-block:: none

        & --> &
        < --> <
        > --> >
        " --> "
        ' --> '
        / --> /  when escape_slash is set to True

    .. versionadded:: 2.1.0
    """
    byte_str = text.encode('utf-8')
    ob = lib.hoedown_buffer_new(OUNIT)
    lib.hoedown_escape_html(ob, byte_str, len(byte_str), int(escape_slash))

    try:
        return to_string(ob)
    finally:
        lib.hoedown_buffer_free(ob)


Instruction:
Implement a Python 3 function for
converting markdown text to HTML.

Response:
def html(text, extensions=0, render_flags=0):
    """
    Convert markdown text to HTML.

    ``extensions`` can be a list or tuple of extensions (e.g.
    ``('fenced-code', 'footnotes', 'strikethrough')``) or an integer
    (e.g. ``EXT_FENCED_CODE | EXT_FOOTNOTES | EXT_STRIKETHROUGH``).

    ``render_flags`` can be a list or tuple of flags (e.g.
    ``('skip-html', 'hard-wrap')``) or an integer
    (e.g. ``HTML_SKIP_HTML | HTML_HARD_WRAP``).
    """
    extensions = args_to_int(extension_map, extensions)
    render_flags = args_to_int(html_flag_map, render_flags)

    ib = lib.hoedown_buffer_new(IUNIT)
    ob = lib.hoedown_buffer_new(OUNIT)
    renderer = lib.hoedown_html_renderer_new(render_flags, 0)
    document = lib.hoedown_document_new(renderer, extensions, 16);

    lib.hoedown_buffer_puts(ib, text.encode('utf-8'))
    lib.hoedown_document_render(document, ob, ib.data, ib.size);
    lib.hoedown_buffer_free(ib);
    lib.hoedown_document_free(document);
    lib.hoedown_html_renderer_free(renderer);

    try:
        return to_string(ob)
    finally:
        lib.hoedown_buffer_free(ob);


Instruction:
Make a summary of the following Python 3 code
def smartypants(text):
    byte_str = text.encode('utf-8')
    ob = lib.hoedown_buffer_new(OUNIT)
    lib.hoedown_html_smartypants(ob, byte_str, len(byte_str))

    try:
        return to_string(ob)
    finally:
        lib.hoedown_buffer_free(ob);

Response: Returns a new node that can be used to render a string of text into HTML entities.


Instruction:
Can you create a Python 3 function that
filters links generated by the autolink extension.

Response:
def autolink(self, raw_url, is_email):
        """
        Filters links generated by the ``autolink`` extension.
        """
        if self.check_url(raw_url):
            url = self.rewrite_url(('mailto:' if is_email else '') + raw_url)
            url = escape_html(url)
            return '%s' % (url, escape_html(raw_url))
        else:
            return escape_html('<%s>' % raw_url)


Instruction:
Explain what the following Python 3 code does
def image(self, raw_url, title='', alt=''):
        if self.check_url(raw_url, is_image_src=True):
            url = self.rewrite_url(raw_url, is_image_src=True)
            maybe_alt = ' alt="%s"' % escape_html(alt) if alt else ''
            maybe_title = ' title="%s"' % escape_html(title) if title else ''
            url = escape_html(url)
            return '<img src="%s"%s%s />' % (url, maybe_alt, maybe_title)
        else:
            return escape_html("![%s](%s)" % (alt, raw_url))

Response: Returns an image tag with optional alt title and alt attributes.


Instruction:
Can you generate the documentation for the following Python 3 function
def check_url(self, url, is_image_src=False):
        return bool(self._allowed_url_re.match(url))

Response: Checks if the given URL is safe.


Instruction:
Explain what the following Python 3 code does
def rewrite_url(self, url, is_image_src=False):
        rewrite = self.img_src_rewrite if is_image_src else self.link_rewrite
        if rewrite:
            return rewrite.format(url=urlquote(url))
        return url

Response: This method is called to rewrite the URL. It returns the URL unchanged if the corresponding attribute is None.


Instruction:
Can you tell what is the following Python 3 function doing
def args_to_int(mapping, argument):
    if isinstance(argument, int):
        if argument == 0:
            return 0
        deprecation('passing extensions and flags as constants is deprecated')
        return argument
    elif isinstance(argument, (tuple, list)):
        return reduce(op.or_, [mapping[n] for n in set(argument) if n in mapping], 0)
    raise TypeError('argument must be a list of strings or an int')

Response: Convert list of strings to an int using a mapping.


Instruction:
How would you explain what the following Python 3 function does
def _pack3(obj, fp, **options):
    global compatibility

    ext_handlers = options.get("ext_handlers")

    if obj is None:
        _pack_nil(obj, fp, options)
    elif ext_handlers and obj.__class__ in ext_handlers:
        _pack_ext(ext_handlers[obj.__class__](obj), fp, options)
    elif isinstance(obj, bool):
        _pack_boolean(obj, fp, options)
    elif isinstance(obj, int):
        _pack_integer(obj, fp, options)
    elif isinstance(obj, float):
        _pack_float(obj, fp, options)
    elif compatibility and isinstance(obj, str):
        _pack_oldspec_raw(obj.encode('utf-8'), fp, options)
    elif compatibility and isinstance(obj, bytes):
        _pack_oldspec_raw(obj, fp, options)
    elif isinstance(obj, str):
        _pack_string(obj, fp, options)
    elif isinstance(obj, bytes):
        _pack_binary(obj, fp, options)
    elif isinstance(obj, (list, tuple)):
        _pack_array(obj, fp, options)
    elif isinstance(obj, dict):
        _pack_map(obj, fp, options)
    elif isinstance(obj, datetime.datetime):
        _pack_ext_timestamp(obj, fp, options)
    elif isinstance(obj, Ext):
        _pack_ext(obj, fp, options)
    elif ext_handlers:
        # Linear search for superclass
        t = next((t for t in ext_handlers.keys() if isinstance(obj, t)), None)
        if t:
            _pack_ext(ext_handlers[t](obj), fp, options)
        else:
            raise UnsupportedTypeException(
                "unsupported type: %s" % str(type(obj)))
    else:
        raise UnsupportedTypeException(
            "unsupported type: %s" % str(type(obj)))

Response: Serializes a Python object into MessagePack bytes.


Instruction:
Can you generate the documentation for the following Python 3 function
def _get_task_target():
  # Break circular dependency.
  # pylint: disable=g-import-not-at-top
  import pipeline
  if pipeline._TEST_MODE:
    return None

  # Further protect against test cases that doesn't set env vars
  # propertly.
  if ("CURRENT_VERSION_ID" not in os.environ or
      "CURRENT_MODULE_ID" not in os.environ):
    logging.warning("Running Pipeline in non TEST_MODE but important "
                    "env vars are not set.")
    return None

  version = os.environ["CURRENT_VERSION_ID"].split(".")[0]
  module = os.environ["CURRENT_MODULE_ID"]
  return "%s.%s" % (version, module)

Response: Get the default target for a pipeline task.


Instruction:
Explain what the following Python 3 code does
def for_name(fq_name, recursive=False):
  fq_name = str(fq_name)
  module_name = __name__
  short_name = fq_name

  if fq_name.rfind(".") >= 0:
    (module_name, short_name) = (fq_name[:fq_name.rfind(".")],
                                 fq_name[fq_name.rfind(".") + 1:])

  try:
    result = __import__(module_name, None, None, [short_name])
    return result.__dict__[short_name]
  except KeyError:
    # If we're recursively inside a for_name() chain, then we want to raise
    # this error as a key error so we can report the actual source of the
    # problem. If we're *not* recursively being called, that means the
    # module was found and the specific item could not be loaded, and thus
    # we want to raise an ImportError directly.
    if recursive:
      raise
    else:
      raise ImportError("Could not find '%s' on path '%s'" % (
                        short_name, module_name))
  except ImportError, e:
    # module_name is not actually a module. Try for_name for it to figure
    # out what's this.
    try:
      module = for_name(module_name, recursive=True)
      if hasattr(module, short_name):
        return getattr(module, short_name)
      else:
        # The module was found, but the function component is missing.
        raise KeyError()
    except KeyError:
      raise ImportError("Could not find '%s' on path '%s'" % (
                        short_name, module_name))
    except ImportError:
      # This means recursive import attempts failed, thus we will raise the
      # first ImportError we encountered, since it's likely the most accurate.
      pass
    # Raise the original import error that caused all of this, since it is
    # likely the real cause of the overall problem.
    raise

Response: Find class or function or method specified by its fully qualified name.


Instruction:
Implement a Python 3 function for
returning true if the object is a user - defined generator function.

Response:
def is_generator_function(obj):
  """Return true if the object is a user-defined generator function.

  Generator function objects provides same attributes as functions.
  See isfunction.__doc__ for attributes listing.

  Adapted from Python 2.6.

  Args:
    obj: an object to test.

  Returns:
    true if the object is generator function.
  """
  CO_GENERATOR = 0x20
  return bool(((inspect.isfunction(obj) or inspect.ismethod(obj)) and
               obj.func_code.co_flags & CO_GENERATOR))


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _register_json_primitive(object_type, encoder, decoder):
  global _TYPE_TO_ENCODER
  global _TYPE_NAME_TO_DECODER
  if object_type not in _TYPE_TO_ENCODER:
    _TYPE_TO_ENCODER[object_type] = encoder
    _TYPE_NAME_TO_DECODER[object_type.__name__] = decoder

Response: Extend what Pipeline can serialize.


Instruction:
Make a summary of the following Python 3 code
def _dict_to_obj(self, d):
    if JsonEncoder.TYPE_ID not in d:
      return d

    type_name = d.pop(JsonEncoder.TYPE_ID)
    if type_name in _TYPE_NAME_TO_DECODER:
      decoder = _TYPE_NAME_TO_DECODER[type_name]
      return decoder(d)
    else:
      raise TypeError("Invalid type %s.", type_name)

Response: Converts a dictionary of json object to a Python object.


Instruction:
Given the following Python 3 function, write the documentation
def _short_repr(obj):
  stringified = pprint.saferepr(obj)
  if len(stringified) > 200:
    return '%s... (%d bytes)' % (stringified[:200], len(stringified))
  return stringified

Response: Helper function returns a truncated repr of an object.


Instruction:
Here you have a function in Python 3, explain what it does
def _write_json_blob(encoded_value, pipeline_id=None):

  default_bucket = app_identity.get_default_gcs_bucket_name()
  if default_bucket is None:
    raise Exception(
      "No default cloud storage bucket has been set for this application. "
      "This app was likely created before v1.9.0, please see: "
      "https://cloud.google.com/appengine/docs/php/googlestorage/setup")

  path_components = ['/', default_bucket, "appengine_pipeline"]
  if pipeline_id:
    path_components.append(pipeline_id)
  path_components.append(uuid.uuid4().hex)
  # Use posixpath to get a / even if we're running on windows somehow
  file_name = posixpath.join(*path_components)
  with cloudstorage.open(file_name, 'w', content_type='application/json') as f:
    for start_index in xrange(0, len(encoded_value), _MAX_JSON_SIZE):
      end_index = start_index + _MAX_JSON_SIZE
      f.write(encoded_value[start_index:end_index])

  key_str = blobstore.create_gs_key("/gs" + file_name)
  logging.debug("Created blob for filename = %s gs_key = %s", file_name, key_str)
  return blobstore.BlobKey(key_str)

Response: Writes a JSON encoded value to a Cloud Storage File.


Instruction:
Can you generate the documentation for the following Python 3 function
def _dereference_args(pipeline_name, args, kwargs):
  lookup_slots = set()
  for arg in itertools.chain(args, kwargs.itervalues()):
    if arg['type'] == 'slot':
      lookup_slots.add(db.Key(arg['slot_key']))

  slot_dict = {}
  for key, slot_record in zip(lookup_slots, db.get(lookup_slots)):
    if slot_record is None or slot_record.status != _SlotRecord.FILLED:
      raise SlotNotFilledError(
          'Slot "%s" missing its value. From %s(*args=%s, **kwargs=%s)' %
          (key, pipeline_name, _short_repr(args), _short_repr(kwargs)))
    slot_dict[key] = slot_record.value

  arg_list = []
  for current_arg in args:
    if current_arg['type'] == 'slot':
      arg_list.append(slot_dict[db.Key(current_arg['slot_key'])])
    elif current_arg['type'] == 'value':
      arg_list.append(current_arg['value'])
    else:
      raise UnexpectedPipelineError('Unknown parameter type: %r' % current_arg)

  kwarg_dict = {}
  for key, current_arg in kwargs.iteritems():
    if current_arg['type'] == 'slot':
      kwarg_dict[key] = slot_dict[db.Key(current_arg['slot_key'])]
    elif current_arg['type'] == 'value':
      kwarg_dict[key] = current_arg['value']
    else:
      raise UnexpectedPipelineError('Unknown parameter type: %r' % current_arg)

  return (arg_list, kwarg_dict)

Response: Dereferences a Pipeline s arguments that are slots validating them.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _generate_args(pipeline, future, queue_name, base_path):
  params = {
      'args': [],
      'kwargs': {},
      'after_all': [],
      'output_slots': {},
      'class_path': pipeline._class_path,
      'queue_name': queue_name,
      'base_path': base_path,
      'backoff_seconds': pipeline.backoff_seconds,
      'backoff_factor': pipeline.backoff_factor,
      'max_attempts': pipeline.max_attempts,
      'task_retry': pipeline.task_retry,
      'target': pipeline.target,
  }
  dependent_slots = set()

  arg_list = params['args']
  for current_arg in pipeline.args:
    if isinstance(current_arg, PipelineFuture):
      current_arg = current_arg.default
    if isinstance(current_arg, Slot):
      arg_list.append({'type': 'slot', 'slot_key': str(current_arg.key)})
      dependent_slots.add(current_arg.key)
    else:
      arg_list.append({'type': 'value', 'value': current_arg})

  kwarg_dict = params['kwargs']
  for name, current_arg in pipeline.kwargs.iteritems():
    if isinstance(current_arg, PipelineFuture):
      current_arg = current_arg.default
    if isinstance(current_arg, Slot):
      kwarg_dict[name] = {'type': 'slot', 'slot_key': str(current_arg.key)}
      dependent_slots.add(current_arg.key)
    else:
      kwarg_dict[name] = {'type': 'value', 'value': current_arg}

  after_all = params['after_all']
  for other_future in future._after_all_pipelines:
    slot_key = other_future._output_dict['default'].key
    after_all.append(str(slot_key))
    dependent_slots.add(slot_key)

  output_slots = params['output_slots']
  output_slot_keys = set()
  for name, slot in future._output_dict.iteritems():
    output_slot_keys.add(slot.key)
    output_slots[name] = str(slot.key)

  params_encoded = json.dumps(params, cls=mr_util.JsonEncoder)
  params_text = None
  params_blob = None
  if len(params_encoded) > _MAX_JSON_SIZE:
    params_blob = _write_json_blob(params_encoded, pipeline.pipeline_id)
  else:
    params_text = params_encoded

  return dependent_slots, output_slot_keys, params_text, params_blob

Response: Generate the arguments needed to describe a Pipeline.


Instruction:
How would you explain what the following Python 3 function does
def _get_timestamp_ms(when):
  if when is None:
    return None
  ms_since_epoch = float(time.mktime(when.utctimetuple()) * 1000.0)
  ms_since_epoch += when.microsecond / 1000.0
  return int(ms_since_epoch)

Response: Converts a datetime. datetime instance to integer milliseconds since the epoch.


Instruction:
Given the following Python 3 function, write the documentation
def _get_internal_status(pipeline_key=None,
                         pipeline_dict=None,
                         slot_dict=None,
                         barrier_dict=None,
                         status_dict=None):
  """Gets the UI dictionary of a pipeline from a set of status dictionaries.

  Args:
    pipeline_key: The key of the pipeline to lookup.
    pipeline_dict: Dictionary mapping pipeline db.Key to _PipelineRecord.
      Default is an empty dictionary.
    slot_dict: Dictionary mapping slot db.Key to _SlotRecord.
      Default is an empty dictionary.
    barrier_dict: Dictionary mapping barrier db.Key to _BarrierRecord.
      Default is an empty dictionary.
    status_dict: Dictionary mapping status record db.Key to _StatusRecord.
      Default is an empty dictionary.

  Returns:
    Dictionary with the keys:
      classPath: The pipeline function being run.
      args: List of positional argument slot dictionaries.
      kwargs: Dictionary of keyword argument slot dictionaries.
      outputs: Dictionary of output slot dictionaries.
      children: List of child pipeline IDs.
      queueName: Queue on which this pipeline is running.
      afterSlotKeys: List of Slot Ids after which this pipeline runs.
      currentAttempt: Number of the current attempt, starting at 1.
      maxAttempts: Maximum number of attempts before aborting.
      backoffSeconds: Constant factor for backoff before retrying.
      backoffFactor: Exponential factor for backoff before retrying.
      status: Current status of the pipeline.
      startTimeMs: When this pipeline ran or will run due to retries, if present.
      endTimeMs: When this pipeline finalized, if present.
      lastRetryMessage: Why the pipeline failed during the last retry, if there
        was a failure; may be empty.
      abortMessage: For root pipelines, why the pipeline was aborted if it was
        aborted; may be empty.

    Dictionary will contain these keys if explicit status is set:
      statusTimeMs: When the status was set as milliseconds since the epoch.
      statusMessage: Status message, if present.
      statusConsoleUrl: The relative URL for the console of this pipeline.
      statusLinks: Dictionary mapping human-readable names to relative URLs
        for related URLs to this pipeline.

  Raises:
    PipelineStatusError if any input is bad.
  """
  if pipeline_dict is None:
    pipeline_dict = {}
  if slot_dict is None:
    slot_dict = {}
  if barrier_dict is None:
    barrier_dict = {}
  if status_dict is None:
    status_dict = {}

  pipeline_record = pipeline_dict.get(pipeline_key)
  if pipeline_record is None:
    raise PipelineStatusError(
        'Could not find pipeline ID "%s"' % pipeline_key.name())

  params = pipeline_record.params
  root_pipeline_key = \
      _PipelineRecord.root_pipeline.get_value_for_datastore(pipeline_record)
  default_slot_key = db.Key(params['output_slots']['default'])
  start_barrier_key = db.Key.from_path(
      _BarrierRecord.kind(), _BarrierRecord.START, parent=pipeline_key)
  finalize_barrier_key = db.Key.from_path(
      _BarrierRecord.kind(), _BarrierRecord.FINALIZE, parent=pipeline_key)
  status_record_key = db.Key.from_path(
      _StatusRecord.kind(), pipeline_key.name())

  start_barrier = barrier_dict.get(start_barrier_key)
  finalize_barrier = barrier_dict.get(finalize_barrier_key)
  default_slot = slot_dict.get(default_slot_key)
  status_record = status_dict.get(status_record_key)
  if finalize_barrier is None:
    raise PipelineStatusError(
        'Finalization barrier missing for pipeline ID "%s"' %
        pipeline_key.name())
  if default_slot is None:
    raise PipelineStatusError(
        'Default output slot with key=%s missing for pipeline ID "%s"' % (
        default_slot_key, pipeline_key.name()))

  output = {
    'classPath': pipeline_record.class_path,
    'args': list(params['args']),
    'kwargs': params['kwargs'].copy(),
    'outputs': params['output_slots'].copy(),
    'children': [key.name() for key in pipeline_record.fanned_out],
    'queueName': params['queue_name'],
    'afterSlotKeys': [str(key) for key in params['after_all']],
    'currentAttempt': pipeline_record.current_attempt + 1,
    'maxAttempts': pipeline_record.max_attempts,
    'backoffSeconds': pipeline_record.params['backoff_seconds'],
    'backoffFactor': pipeline_record.params['backoff_factor'],
  }

  # TODO(user): Truncate args, kwargs, and outputs to < 1MB each so we
  # can reasonably return the whole tree of pipelines and their outputs.
  # Coerce each value to a string to truncate if necessary. For now if the
  # params are too big it will just cause the whole status page to break.

  # Fix the key names in parameters to match JavaScript style.
  for value_dict in itertools.chain(
      output['args'], output['kwargs'].itervalues()):
    if 'slot_key' in value_dict:
      value_dict['slotKey'] = value_dict.pop('slot_key')

  # Figure out the pipeline's status.
  if pipeline_record.status in (_PipelineRecord.WAITING, _PipelineRecord.RUN):
    if default_slot.status == _SlotRecord.FILLED:
      status = 'finalizing'
    elif (pipeline_record.status == _PipelineRecord.WAITING and
          pipeline_record.next_retry_time is not None):
      status = 'retry'
    elif start_barrier and start_barrier.status == _BarrierRecord.WAITING:
      # start_barrier will be missing for root pipelines
      status = 'waiting'
    else:
      status = 'run'
  elif pipeline_record.status == _PipelineRecord.DONE:
    status = 'done'
  elif pipeline_record.status == _PipelineRecord.ABORTED:
    status = 'aborted'

  output['status'] = status

  if status_record:
    output['statusTimeMs'] = _get_timestamp_ms(status_record.status_time)
    if status_record.message:
      output['statusMessage'] = status_record.message
    if status_record.console_url:
      output['statusConsoleUrl'] = status_record.console_url
    if status_record.link_names:
      output['statusLinks'] = dict(
          zip(status_record.link_names, status_record.link_urls))

  # Populate status-depenedent fields.
  if status in ('run', 'finalizing', 'done', 'retry'):
    if pipeline_record.next_retry_time is not None:
      output['startTimeMs'] = _get_timestamp_ms(pipeline_record.next_retry_time)
    elif start_barrier:
      # start_barrier will be missing for root pipelines
      output['startTimeMs'] = _get_timestamp_ms(start_barrier.trigger_time)
    elif pipeline_record.start_time:
      # Assume this pipeline ran immediately upon spawning with no
      # start barrier or it's the root pipeline.
      output['startTimeMs'] = _get_timestamp_ms(pipeline_record.start_time)

  if status in ('finalizing',):
    output['endTimeMs'] = _get_timestamp_ms(default_slot.fill_time)

  if status in ('done',):
    output['endTimeMs'] = _get_timestamp_ms(pipeline_record.finalized_time)

  if pipeline_record.next_retry_time is not None:
    output['lastRetryMessage'] = pipeline_record.retry_message

  if pipeline_record.abort_message:
    output['abortMessage'] = pipeline_record.abort_message

  return output

Response: Internal helper function for _get_pipeline_status.


Instruction:
Explain what the following Python 3 code does
def _get_internal_slot(slot_key=None,
                       filler_pipeline_key=None,
                       slot_dict=None):
  """Gets information about a _SlotRecord for display in UI.

  Args:
    slot_key: The db.Key of the slot to fetch.
    filler_pipeline_key: In the case the slot has not yet been filled, assume
      that the given db.Key (for a _PipelineRecord) will be the filler of
      the slot in the future.
    slot_dict: The slot JSON dictionary.

  Returns:
    Dictionary with the keys:
      status: Slot status: 'filled' or 'waiting'
      fillTimeMs: Time in milliseconds since the epoch of when it was filled.
      value: The current value of the slot, which is a slot's JSON dictionary.
      fillerPipelineId: The pipeline ID of what stage has or should fill
        this slot.

  Raises:
    PipelineStatusError if any input is bad.
  """
  if slot_dict is None:
    slot_dict = {}

  slot_record = slot_dict.get(slot_key)
  if slot_record is None:
    raise PipelineStatusError(
        'Could not find data for output slot key "%s".' % slot_key)

  output = {}
  if slot_record.status == _SlotRecord.FILLED:
    output['status'] = 'filled'
    output['fillTimeMs'] = _get_timestamp_ms(slot_record.fill_time)
    output['value'] = slot_record.value
    filler_pipeline_key = (
        _SlotRecord.filler.get_value_for_datastore(slot_record))
  else:
    output['status'] = 'waiting'

  if filler_pipeline_key:
    output['fillerPipelineId'] = filler_pipeline_key.name()

  return output

Response: Returns a dictionary of _SlotRecord objects for display in UI.


Instruction:
Here you have a function in Python 3, explain what it does
def get_status_tree(root_pipeline_id):
  root_pipeline_key = db.Key.from_path(_PipelineRecord.kind(), root_pipeline_id)
  root_pipeline_record = db.get(root_pipeline_key)
  if root_pipeline_record is None:
    raise PipelineStatusError(
        'Could not find pipeline ID "%s"' % root_pipeline_id)

  # If the supplied root_pipeline_id is not actually the root pipeline that's
  # okay. We'll find the real root and override the value they passed in.
  actual_root_key = _PipelineRecord.root_pipeline.get_value_for_datastore(
      root_pipeline_record)
  if actual_root_key != root_pipeline_key:
    root_pipeline_key = actual_root_key
    root_pipeline_id = root_pipeline_key.id_or_name()
    root_pipeline_record = db.get(root_pipeline_key)
    if not root_pipeline_record:
      raise PipelineStatusError(
          'Could not find pipeline ID "%s"' % root_pipeline_id)

  # Run all queries asynchronously.
  queries = {}
  for model in (_PipelineRecord, _SlotRecord, _BarrierRecord, _StatusRecord):
    queries[model] = model.all().filter(
        'root_pipeline =', root_pipeline_key).run(batch_size=1000)

  found_pipeline_dict = dict(
      (stage.key(), stage) for stage in queries[_PipelineRecord])
  found_slot_dict = dict(
      (slot.key(), slot) for slot in queries[_SlotRecord])
  found_barrier_dict = dict(
      (barrier.key(), barrier) for barrier in queries[_BarrierRecord])
  found_status_dict = dict(
      (status.key(), status) for status in queries[_StatusRecord])

  # Breadth-first traversal of _PipelineRecord instances by following
  # _PipelineRecord.fanned_out property values.
  valid_pipeline_keys = set([root_pipeline_key])
  slot_filler_dict = {}  # slot_key to pipeline_key
  expand_stack = [root_pipeline_record]
  while expand_stack:
    old_stack = expand_stack
    expand_stack = []
    for pipeline_record in old_stack:
      for child_pipeline_key in pipeline_record.fanned_out:
        # This will let us prune off those pipelines which were allocated in
        # the Datastore but were never run due to mid-flight task failures.
        child_pipeline_record = found_pipeline_dict.get(child_pipeline_key)
        if child_pipeline_record is None:
          raise PipelineStatusError(
              'Pipeline ID "%s" points to child ID "%s" which does not exist.'
              % (pipeline_record.key().name(), child_pipeline_key.name()))
        expand_stack.append(child_pipeline_record)
        valid_pipeline_keys.add(child_pipeline_key)

        # Figure out the deepest pipeline that's responsible for outputting to
        # a particular _SlotRecord, so we can report which pipeline *should*
        # be the filler.
        child_outputs = child_pipeline_record.params['output_slots']
        for output_slot_key in child_outputs.itervalues():
          slot_filler_dict[db.Key(output_slot_key)] = child_pipeline_key

  output = {
    'rootPipelineId': root_pipeline_id,
    'slots': {},
    'pipelines': {},
  }

  for pipeline_key in found_pipeline_dict.keys():
    if pipeline_key not in valid_pipeline_keys:
      continue
    output['pipelines'][pipeline_key.name()] = _get_internal_status(
        pipeline_key=pipeline_key,
        pipeline_dict=found_pipeline_dict,
        slot_dict=found_slot_dict,
        barrier_dict=found_barrier_dict,
        status_dict=found_status_dict)

  for slot_key, filler_pipeline_key in slot_filler_dict.iteritems():
    output['slots'][str(slot_key)] = _get_internal_slot(
        slot_key=slot_key,
        filler_pipeline_key=filler_pipeline_key,
        slot_dict=found_slot_dict)

  return output

Response: Gets the full status tree of a pipeline.


Instruction:
How would you code a function in Python 3 to
return the class paths of all Pipelines defined in alphabetical order.

Response:
def get_pipeline_names():
  """Returns the class paths of all Pipelines defined in alphabetical order."""
  class_path_set = set()
  for cls in _PipelineMeta._all_classes:
      if cls.class_path is not None:
        class_path_set.add(cls.class_path)
  return sorted(class_path_set)


Instruction:
Can you create a Python 3 function that
returns a list of Pipelines with the given class_path and cursor.

Response:
def get_root_list(class_path=None, cursor=None, count=50):
  """Gets a list root Pipelines.

  Args:
    class_path: Optional. If supplied, only return root Pipelines with the
      given class_path. By default all root pipelines are returned.
    cursor: Optional. When supplied, the cursor returned from the last call to
      get_root_list which indicates where to pick up.
    count: How many pipeline returns to return.

  Returns:
    Dictionary with the keys:
      pipelines: The list of Pipeline records in the same format as
        returned by get_status_tree, but with only the roots listed.
      cursor: Cursor to pass back to this function to resume the query. Will
        only be present if there is another page of results.

  Raises:
    PipelineStatusError if any input is bad.
  """
  query = _PipelineRecord.all(cursor=cursor)
  if class_path:
    query.filter('class_path =', class_path)
  query.filter('is_root_pipeline =', True)
  query.order('-start_time')

  root_list = query.fetch(count)

  fetch_list = []
  for pipeline_record in root_list:
    fetch_list.append(db.Key(pipeline_record.params['output_slots']['default']))
    fetch_list.append(db.Key.from_path(
        _BarrierRecord.kind(), _BarrierRecord.FINALIZE,
        parent=pipeline_record.key()))
    fetch_list.append(db.Key.from_path(
        _StatusRecord.kind(), pipeline_record.key().name()))

  pipeline_dict = dict((stage.key(), stage) for stage in root_list)
  slot_dict = {}
  barrier_dict = {}
  status_dict = {}
  for entity in db.get(fetch_list):
    if isinstance(entity, _BarrierRecord):
      barrier_dict[entity.key()] = entity
    elif isinstance(entity, _SlotRecord):
      slot_dict[entity.key()] = entity
    elif isinstance(entity, _StatusRecord):
      status_dict[entity.key()] = entity

  results = []
  for pipeline_record in root_list:
    try:
      output = _get_internal_status(
          pipeline_record.key(),
          pipeline_dict=pipeline_dict,
          slot_dict=slot_dict,
          barrier_dict=barrier_dict,
          status_dict=status_dict)
      output['pipelineId'] = pipeline_record.key().name()
      results.append(output)
    except PipelineStatusError, e:
      output = {'status': e.message}
      output['classPath'] = ''
      output['pipelineId'] = pipeline_record.key().name()
      results.append(output)

  result_dict = {}
  cursor = query.cursor()
  query.with_cursor(cursor)
  if query.get(keys_only=True):
    result_dict.update(cursor=cursor)
  result_dict.update(pipelines=results)
  return result_dict


Instruction:
Can you tell what is the following Python 3 function doing
def create_handlers_map(prefix='.*'):
  return [
      (prefix + '/output', _BarrierHandler),
      (prefix + '/run', _PipelineHandler),
      (prefix + '/finalized', _PipelineHandler),
      (prefix + '/cleanup', _CleanupHandler),
      (prefix + '/abort', _PipelineHandler),
      (prefix + '/fanout', _FanoutHandler),
      (prefix + '/fanout_abort', _FanoutAbortHandler),
      (prefix + '/callback', _CallbackHandler),
      (prefix + '/rpc/tree', status_ui._TreeStatusHandler),
      (prefix + '/rpc/class_paths', status_ui._ClassPathListHandler),
      (prefix + '/rpc/list', status_ui._RootListHandler),
      (prefix + '(/.+)', status_ui._StatusUiHandler),
      ]

Response: Create a new handlers map.


Instruction:
Create a Python 3 function to
return the current value of this slot.

Response:
def value(self):
    """Returns the current value of this slot.

    Returns:
      The value of the slot (a serializable Python type).

    Raises:
      SlotNotFilledError if the value hasn't been filled yet.
    """
    if not self.filled:
      raise SlotNotFilledError('Slot with name "%s", key "%s" not yet filled.'
                               % (self.name, self.key))
    return self._value


Instruction:
Can you write a function in Python 3 where it
returns the pipeline ID that filled this slot s value.

Response:
def filler(self):
    """Returns the pipeline ID that filled this slot's value.

    Returns:
      A string that is the pipeline ID.

    Raises:
      SlotNotFilledError if the value hasn't been filled yet.
    """
    if not self.filled:
      raise SlotNotFilledError('Slot with name "%s", key "%s" not yet filled.'
                               % (self.name, self.key))
    return self._filler_pipeline_key.name()


Instruction:
Implement a Python 3 function for
returning when the slot was filled.

Response:
def fill_datetime(self):
    """Returns when the slot was filled.

    Returns:
      A datetime.datetime.

    Raises:
      SlotNotFilledError if the value hasn't been filled yet.
    """
    if not self.filled:
      raise SlotNotFilledError('Slot with name "%s", key "%s" not yet filled.'
                               % (self.name, self.key))
    return self._fill_datetime


Instruction:
Can you write a function in Python 3 where it
sets the value of this slot based on its corresponding _SlotRecord.

Response:
def _set_value(self, slot_record):
    """Sets the value of this slot based on its corresponding _SlotRecord.

    Does nothing if the slot has not yet been filled.

    Args:
      slot_record: The _SlotRecord containing this Slot's value.
    """
    if slot_record.status == _SlotRecord.FILLED:
      self.filled = True
      self._filler_pipeline_key = _SlotRecord.filler.get_value_for_datastore(
          slot_record)
      self._fill_datetime = slot_record.fill_time
      self._value = slot_record.value


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _inherit_outputs(self,
                       pipeline_name,
                       already_defined,
                       resolve_outputs=False):
    """Inherits outputs from a calling Pipeline.

    Args:
      pipeline_name: The Pipeline class name (used for debugging).
      already_defined: Maps output name to stringified db.Key (of _SlotRecords)
        of any exiting output slots to be inherited by this future.
      resolve_outputs: When True, this method will dereference all output slots
        before returning back to the caller, making those output slots' values
        available.

    Raises:
      UnexpectedPipelineError when resolve_outputs is True and any of the output
      slots could not be retrived from the Datastore.
    """
    for name, slot_key in already_defined.iteritems():
      if not isinstance(slot_key, db.Key):
        slot_key = db.Key(slot_key)

      slot = self._output_dict.get(name)
      if slot is None:
        if self._strict:
          raise UnexpectedPipelineError(
              'Inherited output named "%s" must be filled but '
              'not declared for pipeline class "%s"' % (name, pipeline_name))
        else:
          self._output_dict[name] = Slot(name=name, slot_key=slot_key)
      else:
        slot.key = slot_key
        slot._exists = True

    if resolve_outputs:
      slot_key_dict = dict((s.key, s) for s in self._output_dict.itervalues())
      all_slots = db.get(slot_key_dict.keys())
      for slot, slot_record in zip(slot_key_dict.itervalues(), all_slots):
        if slot_record is None:
          raise UnexpectedPipelineError(
              'Inherited output named "%s" for pipeline class "%s" is '
              'missing its Slot in the datastore: "%s"' %
              (slot.name, pipeline_name, slot.key))
        slot = slot_key_dict[slot_record.key()]
        slot._set_value(slot_record)

Response: Inherits outputs from a calling Pipeline.


Instruction:
Make a summary of the following Python 3 code
def from_id(cls, pipeline_id, resolve_outputs=True, _pipeline_record=None):
    pipeline_record = _pipeline_record

    # Support pipeline IDs and idempotence_keys that are not unicode.
    if not isinstance(pipeline_id, unicode):
      try:
        pipeline_id = pipeline_id.encode('utf-8')
      except UnicodeDecodeError:
        pipeline_id = hashlib.sha1(pipeline_id).hexdigest()

    pipeline_key = db.Key.from_path(_PipelineRecord.kind(), pipeline_id)

    if pipeline_record is None:
      pipeline_record = db.get(pipeline_key)
    if pipeline_record is None:
      return None

    try:
      pipeline_func_class = mr_util.for_name(pipeline_record.class_path)
    except ImportError, e:
      logging.warning('Tried to find Pipeline %s#%s, but class could '
                      'not be found. Using default Pipeline class instead.',
                      pipeline_record.class_path, pipeline_id)
      pipeline_func_class = cls

    params = pipeline_record.params
    arg_list, kwarg_dict = _dereference_args(
        pipeline_record.class_path, params['args'], params['kwargs'])
    outputs = PipelineFuture(pipeline_func_class.output_names)
    outputs._inherit_outputs(
        pipeline_record.class_path,
        params['output_slots'],
        resolve_outputs=resolve_outputs)

    stage = pipeline_func_class(*arg_list, **kwarg_dict)
    stage.backoff_seconds = params['backoff_seconds']
    stage.backoff_factor = params['backoff_factor']
    stage.max_attempts = params['max_attempts']
    stage.task_retry = params['task_retry']
    stage.target = params.get('target')  # May not be defined for old Pipelines
    stage._current_attempt = pipeline_record.current_attempt
    stage._set_values_internal(
        _PipelineContext('', params['queue_name'], params['base_path']),
        pipeline_key,
        _PipelineRecord.root_pipeline.get_value_for_datastore(pipeline_record),
        outputs,
        pipeline_record.status)
    return stage

Response: Returns an instance corresponding to a Pipeline with the given ID.


Instruction:
Make a summary of the following Python 3 code
def start(self,
            idempotence_key='',
            queue_name='default',
            base_path='/_ah/pipeline',
            return_task=False,
            countdown=None,
            eta=None):
    """Starts a new instance of this pipeline.

    Args:
      idempotence_key: The ID to use for this Pipeline and throughout its
        asynchronous workflow to ensure the operations are idempotent. If
        empty a starting key will be automatically assigned.
      queue_name: What queue this Pipeline's workflow should execute on.
      base_path: The relative URL path to where the Pipeline API is
        mounted for access by the taskqueue API or external requests.
      return_task: When True, a task to start this pipeline will be returned
        instead of submitted, allowing the caller to start off this pipeline
        as part of a separate transaction (potentially leaving this newly
        allocated pipeline's datastore entities in place if that separate
        transaction fails for any reason).
      countdown: Time in seconds into the future that this Task should execute.
        Defaults to zero.
      eta: A datetime.datetime specifying the absolute time at which the task
        should be executed. Must not be specified if 'countdown' is specified.
        This may be timezone-aware or timezone-naive. If None, defaults to now.
        For pull tasks, no worker will be able to lease this task before the
        time indicated by eta.

    Returns:
      A taskqueue.Task instance if return_task was True. This task will *not*
      have a name, thus to ensure reliable execution of your pipeline you
      should add() this task as part of a separate Datastore transaction.

    Raises:
      PipelineExistsError if the pipeline with the given idempotence key exists.
      PipelineSetupError if the pipeline could not start for any other reason.
    """
    if not idempotence_key:
      idempotence_key = uuid.uuid4().hex
    elif not isinstance(idempotence_key, unicode):
      try:
        idempotence_key.encode('utf-8')
      except UnicodeDecodeError:
        idempotence_key = hashlib.sha1(idempotence_key).hexdigest()

    pipeline_key = db.Key.from_path(_PipelineRecord.kind(), idempotence_key)
    context = _PipelineContext('', queue_name, base_path)
    future = PipelineFuture(self.output_names, force_strict=True)
    try:
      self._set_values_internal(
          context, pipeline_key, pipeline_key, future, _PipelineRecord.WAITING)
      return context.start(
          self, return_task=return_task, countdown=countdown, eta=eta)
    except Error:
      # Pass through exceptions that originate in this module.
      raise
    except Exception, e:
      # Re-type any exceptions that were raised in dependent methods.
      raise PipelineSetupError('Error starting %s#%s: %s' % (
          self, idempotence_key, str(e)))

Response: Starts a new taskqueue. Task instance for this pipeline.


Instruction:
Make a summary of the following Python 3 code
def retry(self, retry_message=''):
    if not self.async:
      raise UnexpectedPipelineError(
          'May only call retry() method for asynchronous pipelines.')
    if self.try_cancel():
      self._context.transition_retry(self._pipeline_key, retry_message)
      return True
    else:
      return False

Response: Forces a currently running asynchronous pipeline to retry.


Instruction:
Explain what the following Python 3 code does
def abort(self, abort_message=''):
    # TODO: Use thread-local variable to enforce that this is not called
    # while a pipeline is executing in the current thread.
    if (self.async and self._root_pipeline_key == self._pipeline_key and
        not self.try_cancel()):
      # Handle the special case where the root pipeline is async and thus
      # cannot be aborted outright.
      return False
    else:
      return self._context.begin_abort(
          self._root_pipeline_key, abort_message=abort_message)

Response: Mark the entire pipeline up to the root as aborted.


Instruction:
Write a Python 3 function that can
fill an output slot required by this Pipeline.

Response:
def fill(self, name_or_slot, value):
    """Fills an output slot required by this Pipeline.

    Args:
      name_or_slot: The name of the slot (a string) or Slot record to fill.
      value: The serializable value to assign to this slot.

    Raises:
      UnexpectedPipelineError if the Slot no longer exists. SlotNotDeclaredError
      if trying to output to a slot that was not declared ahead of time.
    """
    if isinstance(name_or_slot, basestring):
      slot = getattr(self.outputs, name_or_slot)
    elif isinstance(name_or_slot, Slot):
      slot = name_or_slot
    else:
      raise UnexpectedPipelineError(
          'Could not fill invalid output name: %r' % name_or_slot)

    if not slot._exists:
      raise SlotNotDeclaredError(
          'Cannot fill output with name "%s" that was just '
          'declared within the Pipeline context.' % slot.name)

    self._context.fill_slot(self._pipeline_key, slot, value)


Instruction:
Write a Python 3 function that can
set the current status of this pipeline.

Response:
def set_status(self, message=None, console_url=None, status_links=None):
    """Sets the current status of this pipeline.

    This method is purposefully non-transactional. Updates are written to the
    datastore immediately and overwrite all existing statuses.

    Args:
      message: (optional) Overall status message.
      console_url: (optional) Relative URL to use for the "console" of this
        pipeline that displays current progress. When None, no console will
        be displayed.
      status_links: (optional) Dictionary of readable link names to relative
        URLs that should be associated with this pipeline as it runs. These links
        provide convenient access to other dashboards, consoles, etc associated
        with the pipeline.

    Raises:
      PipelineRuntimeError if the status could not be set for any reason.
    """
    if _TEST_MODE:
      logging.info(
          'New status for %s#%s: message=%r, console_url=%r, status_links=%r',
          self, self.pipeline_id, message, console_url, status_links)
      return

    status_key = db.Key.from_path(_StatusRecord.kind(), self.pipeline_id)
    root_pipeline_key = db.Key.from_path(
        _PipelineRecord.kind(), self.root_pipeline_id)
    status_record = _StatusRecord(
        key=status_key, root_pipeline=root_pipeline_key)

    try:
      if message:
        status_record.message = message
      if console_url:
        status_record.console_url = console_url
      if status_links:
        # Alphabeticalize the list.
        status_record.link_names = sorted(
            db.Text(s) for s in status_links.iterkeys())
        status_record.link_urls = [
            db.Text(status_links[name]) for name in status_record.link_names]

      status_record.status_time = datetime.datetime.utcnow()

      status_record.put()
    except Exception, e:
      raise PipelineRuntimeError('Could not set status for %s#%s: %s' %
          (self, self.pipeline_id, str(e)))


Instruction:
Create a Python 3 function to
mark this asynchronous Pipeline as complete.

Response:
def complete(self, default_output=None):
    """Marks this asynchronous Pipeline as complete.

    Args:
      default_output: What value the 'default' output slot should be assigned.

    Raises:
      UnexpectedPipelineError if the slot no longer exists or this method was
      called for a pipeline that is not async.
    """
    # TODO: Enforce that all outputs expected by this async pipeline were
    # filled before this complete() function was called. May required all
    # async functions to declare their outputs upfront.
    if not self.async:
      raise UnexpectedPipelineError(
          'May only call complete() method for asynchronous pipelines.')
    self._context.fill_slot(
        self._pipeline_key, self.outputs.default, default_output)


Instruction:
Can you generate the documentation for the following Python 3 function
def get_callback_url(self, **kwargs):
    # TODO: Support positional parameters.
    if not self.async:
      raise UnexpectedPipelineError(
          'May only call get_callback_url() method for asynchronous pipelines.')
    kwargs['pipeline_id'] = self._pipeline_key.name()
    params = urllib.urlencode(sorted(kwargs.items()))
    return '%s/callback?%s' % (self.base_path, params)

Response: Returns a relative URL for invoking this Pipeline s callback method.


Instruction:
Create a Python 3 function to
return a task that will be executed when this Pipeline is executed.

Response:
def get_callback_task(self, *args, **kwargs):
    """Returns a task for calling back this Pipeline.

    Args:
      params: Keyword argument containing a dictionary of key/value pairs
        that will be passed to the callback when it is executed.
      args, kwargs: Passed to the taskqueue.Task constructor. Use these
        arguments to set the task name (for idempotence), etc.

    Returns:
      A taskqueue.Task instance that must be enqueued by the caller.
    """
    if not self.async:
      raise UnexpectedPipelineError(
          'May only call get_callback_task() method for asynchronous pipelines.')

    params = kwargs.get('params', {})
    kwargs['params'] = params
    params['pipeline_id'] = self._pipeline_key.name()
    kwargs['url'] = self.base_path + '/callback'
    kwargs['method'] = 'POST'
    return taskqueue.Task(*args, **kwargs)


Instruction:
Write a Python 3 function that can
send an email to admins that the pipeline has completed.

Response:
def send_result_email(self, sender=None):
    """Sends an email to admins indicating this Pipeline has completed.

    For developer convenience. Automatically called from finalized for root
    Pipelines that do not override the default action.

    Args:
      sender: (optional) Override the sender's email address.
    """
    status = 'successful'
    if self.was_aborted:
      status = 'aborted'

    app_id = os.environ['APPLICATION_ID']
    shard_index = app_id.find('~')
    if shard_index != -1:
      app_id = app_id[shard_index+1:]

    param_dict = {
        'status': status,
        'app_id': app_id,
        'class_path': self._class_path,
        'pipeline_id': self.root_pipeline_id,
        'base_path': '%s.appspot.com%s' % (app_id, self.base_path),
    }
    subject = (
        'Pipeline %(status)s: App "%(app_id)s", %(class_path)s'
        '#%(pipeline_id)s' % param_dict)
    body = """View the pipeline results here:

http://%(base_path)s/status?root=%(pipeline_id)s

Thanks,

The Pipeline API
""" % param_dict

    html = """<html><body>
<p>View the pipeline results here:</p>

<p><a href="http://%(base_path)s/status?root=%(pipeline_id)s"
>http://%(base_path)s/status?root=%(pipeline_id)s</a></p>

<p>
Thanks,
<br>
The Pipeline API
</p>
</body></html>
""" % param_dict

    if sender is None:
      sender = '%s@%s.appspotmail.com' % (app_id, app_id)
    try:
      self._send_mail(sender, subject, body, html=html)
    except (mail.InvalidSenderError, mail.InvalidEmailError):
      logging.warning('Could not send result email for '
                      'root pipeline ID "%s" from sender "%s"',
                      self.root_pipeline_id, sender)


Instruction:
Create a Python 3 function to
clean up this Pipeline and all Datastore records used for coordination.

Response:
def cleanup(self):
    """Clean up this Pipeline and all Datastore records used for coordination.

    Only works when called on a root pipeline. Child pipelines will ignore
    calls to this method.

    After this method is called, Pipeline.from_id() and related status
    methods will return inconsistent or missing results. This method is
    fire-and-forget and asynchronous.
    """
    if self._root_pipeline_key is None:
      raise UnexpectedPipelineError(
          'Could not cleanup Pipeline with unknown root pipeline ID.')
    if not self.is_root:
      return
    task = taskqueue.Task(
        params=dict(root_pipeline_key=self._root_pipeline_key),
        url=self.base_path + '/cleanup',
        headers={'X-Ae-Pipeline-Key': self._root_pipeline_key})
    taskqueue.Queue(self.queue_name).add(task)


Instruction:
Make a summary of the following Python 3 code
def with_params(self, **kwargs):
    if _TEST_MODE:
      logging.info(
          'Setting runtime parameters for %s#%s: %r',
          self, self.pipeline_id, kwargs)
      return self

    if self.pipeline_id is not None:
      raise UnexpectedPipelineError(
          'May only call with_params() on a Pipeline that has not yet '
          'been scheduled for execution.')

    ALLOWED = ('backoff_seconds', 'backoff_factor', 'max_attempts', 'target')
    for name, value in kwargs.iteritems():
      if name not in ALLOWED:
        raise TypeError('Unexpected keyword: %s=%r' % (name, value))
      setattr(self, name, value)
    return self

Response: Modify the execution parameters of this Pipeline instance before it runs.


Instruction:
Create a Python 3 function to
set the absolute path to this class as a string.

Response:
def _set_class_path(cls, module_dict=sys.modules):
    """Sets the absolute path to this class as a string.

    Used by the Pipeline API to reconstruct the Pipeline sub-class object
    at execution time instead of passing around a serialized function.

    Args:
      module_dict: Used for testing.
    """
    # Do not traverse the class hierarchy fetching the class path attribute.
    found = cls.__dict__.get('_class_path')
    if found is not None:
      return

    # Do not set the _class_path for the base-class, otherwise all children's
    # lookups for _class_path will fall through and return 'Pipeline' above.
    # This situation can happen if users call the generic Pipeline.from_id
    # to get the result of a Pipeline without knowing its specific class.
    if cls is Pipeline:
      return

    class_path = '%s.%s' % (cls.__module__, cls.__name__)
    # When a WSGI handler is invoked as an entry point, any Pipeline class
    # defined in the same file as the handler will get __module__ set to
    # __main__. Thus we need to find out its real fully qualified path.
    if cls.__module__ == '__main__':
      for name, module in module_dict.items():
        if name == '__main__':
          continue
        found = getattr(module, cls.__name__, None)
        if found is cls:
          class_path = '%s.%s' % (name, cls.__name__)
          break
    cls._class_path = class_path


Instruction:
How would you code a function in Python 3 to
set the user - visible values provided as an API by this class.

Response:
def _set_values_internal(self,
                           context,
                           pipeline_key,
                           root_pipeline_key,
                           outputs,
                           result_status):
    """Sets the user-visible values provided as an API by this class.

    Args:
      context: The _PipelineContext used for this Pipeline.
      pipeline_key: The db.Key of this pipeline.
      root_pipeline_key: The db.Key of the root pipeline.
      outputs: The PipelineFuture for this pipeline.
      result_status: The result status of this pipeline.
    """
    self._context = context
    self._pipeline_key = pipeline_key
    self._root_pipeline_key = root_pipeline_key
    self._result_status = result_status
    self.outputs = outputs


Instruction:
How would you explain what the following Python 3 function does
def _callback_internal(self, kwargs):
    logging.debug('Callback %s(*%s, **%s)#%s with params: %r',
                  self._class_path, _short_repr(self.args),
                  _short_repr(self.kwargs), self._pipeline_key.name(), kwargs)
    return self.callback(**kwargs)

Response: Used to execute callbacks on asynchronous pipelines.


Instruction:
Here you have a function in Python 3, explain what it does
def _run_internal(self,
                    context,
                    pipeline_key,
                    root_pipeline_key,
                    caller_output):
    """Used by the Pipeline evaluator to execute this Pipeline."""
    self._set_values_internal(
        context, pipeline_key, root_pipeline_key, caller_output,
        _PipelineRecord.RUN)
    logging.debug('Running %s(*%s, **%s)#%s',
                  self._class_path, _short_repr(self.args),
                  _short_repr(self.kwargs), self._pipeline_key.name())
    return self.run(*self.args, **self.kwargs)

Response: Internal method to execute this Pipeline.


Instruction:
How would you implement a function in Python 3 that
uses by the Pipeline evaluator to finalize this Pipeline.

Response:
def _finalized_internal(self,
                          context,
                          pipeline_key,
                          root_pipeline_key,
                          caller_output,
                          aborted):
    """Used by the Pipeline evaluator to finalize this Pipeline."""
    result_status = _PipelineRecord.RUN
    if aborted:
      result_status = _PipelineRecord.ABORTED

    self._set_values_internal(
        context, pipeline_key, root_pipeline_key, caller_output, result_status)
    logging.debug('Finalizing %s(*%r, **%r)#%s',
                  self._class_path, _short_repr(self.args),
                  _short_repr(self.kwargs), self._pipeline_key.name())
    try:
      self.finalized()
    except NotImplementedError:
      pass


Instruction:
Write a Python 3 script for
adding a future to the list of in - order futures thus far.

Response:
def _add_future(cls, future):
    """Adds a future to the list of in-order futures thus far.

    Args:
      future: The future to add to the list.
    """
    if cls._local._activated:
      cls._local._in_order_futures.add(future)


Instruction:
Write a Python 3 script for
ensuring thread local is initialized.

Response:
def _thread_init(cls):
    """Ensure thread local is initialized."""
    if not hasattr(cls._local, '_in_order_futures'):
      cls._local._in_order_futures = set()
      cls._local._activated = False


Instruction:
Implement a Python 3 function for
constructing a _PipelineContext from the task queue environment.

Response:
def from_environ(cls, environ=os.environ):
    """Constructs a _PipelineContext from the task queue environment."""
    base_path, unused = (environ['PATH_INFO'].rsplit('/', 1) + [''])[:2]
    return cls(
        environ['HTTP_X_APPENGINE_TASKNAME'],
        environ['HTTP_X_APPENGINE_QUEUENAME'],
        base_path)


Instruction:
How would you explain what the following Python 3 function does
def fill_slot(self, filler_pipeline_key, slot, value):
    if not isinstance(filler_pipeline_key, db.Key):
      filler_pipeline_key = db.Key(filler_pipeline_key)

    if _TEST_MODE:
      slot._set_value_test(filler_pipeline_key, value)
    else:
      encoded_value = json.dumps(value,
                                       sort_keys=True,
                                       cls=mr_util.JsonEncoder)
      value_text = None
      value_blob = None
      if len(encoded_value) <= _MAX_JSON_SIZE:
        value_text = db.Text(encoded_value)
      else:
        # The encoded value is too big. Save it as a blob.
        value_blob = _write_json_blob(encoded_value, filler_pipeline_key.name())

      def txn():
        slot_record = db.get(slot.key)
        if slot_record is None:
          raise UnexpectedPipelineError(
              'Tried to fill missing slot "%s" '
              'by pipeline ID "%s" with value: %r'
              % (slot.key, filler_pipeline_key.name(), value))
        # NOTE: Always take the override value here. If down-stream pipelines
        # need a consitent view of all up-stream outputs (meaning, all of the
        # outputs came from the same retry attempt of the upstream pipeline),
        # the down-stream pipeline must also wait for the 'default' output
        # of these up-stream pipelines.
        slot_record.filler = filler_pipeline_key
        slot_record.value_text = value_text
        slot_record.value_blob = value_blob
        slot_record.status = _SlotRecord.FILLED
        slot_record.fill_time = self._gettime()
        slot_record.put()
        task = taskqueue.Task(
            url=self.barrier_handler_path,
            params=dict(
                slot_key=slot.key,
                use_barrier_indexes=True),
            headers={'X-Ae-Slot-Key': slot.key,
                     'X-Ae-Filler-Pipeline-Key': filler_pipeline_key})
        task.add(queue_name=self.queue_name, transactional=True)
      db.run_in_transaction_options(
          db.create_transaction_options(propagation=db.ALLOWED), txn)

    self.session_filled_output_names.add(slot.name)

Response: Fills a slot with the given value.