Instruction:
How would you code a function in Python 3 to
return the DNA melting temp using nearest - neighbor thermodynamics.

Response:
def temp(s, DNA_c=5000.0, Na_c=10.0, Mg_c=20.0, dNTPs_c=10.0, uncorrected=False):
    '''
    Returns the DNA/DNA melting temp using nearest-neighbor thermodynamics.

    This function returns better results than EMBOSS DAN because it uses updated
    thermodynamics values and takes into account initialization parameters from
    the work of SantaLucia (1998).

    Corrects for mono- and divalent cation concentrations.

    Arguments:
    - DNA_c:   DNA concentration [nM]
    - Na_c:    Na+ concentration [mM]
    - Mg_c:    Mg2+ concentration [mM]
    - dNTPs_c: dNTP concentration [mM]
    - correction: correct for cation concentration?
    '''

    R = 1.987    # Universal gas constant (cal/(K*mol))
    s = s.upper()
    dh, ds = _tercorr(s)
    k = DNA_c * 1e-9

    # Adapted from Table 1 in Allawi and SantaLucia (1997).
    # delta H (kcal/mol)
    dh_coeffs = {"AA": -7.9, "TT": -7.9,
                 "AT": -7.2,
                 "TA": -7.2,
                 "CA": -8.5, "TG": -8.5,
                 "GT": -8.4, "AC": -8.4,
                 "CT": -7.8, "AG": -7.8,
                 "GA": -8.2, "TC": -8.2,
                 "CG": -10.6,
                 "GC": -9.8,
                 "GG": -8.0, "CC": -8.0}

    # delta S (eu)
    ds_coeffs = {"AA": -22.2, "TT": -22.2,
                 "AT": -20.4,
                 "TA": -21.3,
                 "CA": -22.7, "TG": -22.7,
                 "GT": -22.4, "AC": -22.4,
                 "CT": -21.0, "AG": -21.0,
                 "GA": -22.2, "TC": -22.2,
                 "CG": -27.2,
                 "GC": -24.4,
                 "GG": -19.9, "CC": -19.9}

    # Multiplies the number of times each nuc pair is in the sequence by the
    # appropriate coefficient, then returns the sum of all the pairs
    dh = dh + \
        sum(_overcount(s, pair) * coeff for pair, coeff in dh_coeffs.items())
    ds = ds + \
        sum(_overcount(s, pair) * coeff for pair, coeff in ds_coeffs.items())

    fgc = len([filter(lambda x: x == 'G' or x == 'C', s)]) / float(len(s))

    # Melting temperature
    tm = (1000 * dh) / (ds + (R * log(k)))

    if uncorrected:
        return tm - 273.15

    MNa = Na_c * 1e-3
    MMg = Mg_c * 1e-3
    MdNTPs = dNTPs_c * 1e-3

    # Free magnesium concentration
    Ka = 3e4  # association constant in biological buffers
    D = (Ka * MdNTPs - Ka * MMg + 1)**2 + (4 * Ka * MMg)
    Fmg = (-(Ka * MdNTPs - Ka * MMg + 1) + sqrt(D)) / (2 * Ka)

    cation_ratio = sqrt(Fmg) / MNa if MNa > 0 else 7.0

    if cation_ratio < 0.22:
        tm = 1 / (
            (1 / tm) +
            ((4.29 * fgc - 3.95) * log(MNa) + 0.94 * log(MNa)**2) * 1e-5)
    else:
        a = 3.92
        d = 1.42
        g = 8.31
        Fmg = MMg
        if cation_ratio < 6.0:
            a = a * (0.843 - 0.352 * sqrt(MNa) * log(MNa))
            d = d * \
                (1.279 - 4.03 * log(MNa) * 1e-3 - 8.03 * log(MNa)**2 * 1e-3)
            g = g * (0.486 - 0.258 * log(MNa) + 5.25 * log(MNa)**3 * 1e-3)
        tm = 1 / (
            (1 / tm) +
            (a - 0.911 * log(Fmg) + fgc * (6.26 + d * log(Fmg)) +
             1 / (2 * (len(s) - 1)) * (-48.2 + 52.5 * log(Fmg) +
                                       g * log(Fmg)**2)) * 1e-5)

    return tm - 273.15


Instruction:
Implement a Python 3 function for
writing a xy_report based on xy data.

Response:
def write_xy_report(odb, path, tags, columns, steps):
  """
  Writes a xy_report based on xy data.
  """
  xyData = [session.XYDataFromHistory(name = columns[i],
                    odb = odb,
                    outputVariableName = tags[i],
                    steps = steps)
            for i in xrange(len(tags))]
  session.xyReportOptions.setValues(numDigits=8, numberFormat=SCIENTIFIC)
  session.writeXYReport(fileName=path, appendMode=OFF, xyData=xyData)


Instruction:
Given the following Python 3 function, write the documentation
def write_field_report(odb, path, label, argiope_class, variable, instance, output_position,
                       step = -1, frame = -1, sortItem='Node Label'):
  """
  Writes a field report and rewrites it in a cleaner format.
  """
  stepKeys = get_steps(odb)
  step = xrange(len(stepKeys))[step]
  frame = xrange(get_frames(odb, stepKeys[step]))[frame]
  nf = NumberFormat(numDigits=9,
                    precision=0,
                    format=SCIENTIFIC)
  session.fieldReportOptions.setValues(
          printTotal=OFF,
          printMinMax=OFF,
          numberFormat=nf)
  leaf = dgo.LeafFromPartInstance(
          partInstanceName = instance)
  session.viewports['Viewport: 1'].odbDisplay.displayGroup.replace(leaf=leaf)
  session.writeFieldReport(
          fileName       = path,
          append         = OFF,
          sortItem       = sortItem,
          odb            = odb,
          step           = step,
          frame          = frame,
          outputPosition = output_position,
          variable       = variable)
  lines = [line.strip() for line in open(path).readlines()]
  isdata = -1
  data = []
  for line in lines:
   if isdata == 1:
     if len(line) == 0:
       isdata -= 1
     else:
       data.append(line)
   elif isdata < 1:
     if line.startswith("--"):
       isdata += 1
  data = "\n".join([",".join(line.split()) for line in data if len(line) != 0])
  # HEADER
  header = str(output_position).lower() + ","
  header += ",".join([v[1] for v in variable[0][2]]) + "\n"
  # METADATA
  metadata = (
          ("label", label),
          ("argiope_class", argiope_class) ,
          ("odb", odb.path),
          ("instance", instance),
          ("position", output_position),
          ("step_num", step),
          ("step_label", stepKeys[step]),
          ("frame", frame),
          ("frame_value", odb.steps[stepKeys[step]].frames[frame].frameValue)
              )
  out = "*METADATA\n{0}\n*DATA\n{1}".format(
        "\n".join(["{0}={1}".format(k, v) for k, v in metadata]),
        header + data)
  open(path, "w").write(out)

Response: Writes a field report to the database.


Instruction:
Can you generate the documentation for the following Python 3 function
def start(dashboards, once, secrets):

    if secrets is None:
        secrets = os.path.join(os.path.expanduser("~"), "/.doodledashboard/secrets")

    try:
        loaded_secrets = try_read_secrets_file(secrets)
    except InvalidSecretsException as err:
        click.echo(get_error_message(err, default="Secrets file is invalid"), err=True)
        raise click.Abort()

    read_configs = ["""
    dashboard:
      display:
        type: console
    """]
    for dashboard_file in dashboards:
        read_configs.append(read_file(dashboard_file))

    dashboard_config = DashboardConfigReader(initialise_component_loader(), loaded_secrets)

    try:
        dashboard = read_dashboard_from_config(dashboard_config, read_configs)
    except YAMLError as err:
        click.echo(get_error_message(err, default="Dashboard configuration is invalid"), err=True)
        raise click.Abort()

    try:
        DashboardValidator().validate(dashboard)
    except ValidationException as err:
        click.echo(get_error_message(err, default="Dashboard configuration is invalid"), err=True)
        raise click.Abort()

    explain_dashboard(dashboard)

    click.echo("Dashboard running...")

    while True:
        try:
            DashboardRunner(dashboard).cycle()
        except SecretNotFound as err:
            click.echo(get_error_message(err, default="Datafeed didn't have required secret"), err=True)
            raise click.Abort()

        if once:
            break

Response: Start a new dashboard from the provided dashboards.


Instruction:
How would you explain what the following Python 3 function does
def view(action, dashboards, secrets):

    if secrets is None:
        secrets = os.path.join(os.path.expanduser("~"), "/.doodledashboard/secrets")

    try:
        loaded_secrets = try_read_secrets_file(secrets)
    except InvalidSecretsException as err:
        click.echo(get_error_message(err, default="Secrets file is invalid"), err=True)
        raise click.Abort()

    dashboard_config = DashboardConfigReader(initialise_component_loader(), loaded_secrets)

    read_configs = [read_file(f) for f in dashboards]
    dashboard = read_dashboard_from_config(dashboard_config, read_configs)

    try:
        messages = DashboardRunner(dashboard).poll_datafeeds()
    except SecretNotFound as err:
        click.echo(get_error_message(err, default="Datafeed didn't have required secret"), err=True)
        raise click.Abort()

    cli_output = {"source-data": messages}

    if action == "notifications":
        cli_output["notifications"] = []

        for notification in dashboard.notifications:
            notification_output = notification.create(messages)

            filtered_messages = messages

            if isinstance(notification, FilteredNotification):
                filtered_messages = notification.filter_messages(messages)

            cli_output["notifications"].append({
                "filtered-messages": filtered_messages,
                "notification": str(notification_output)
            })
    json_output = json.dumps(cli_output, sort_keys=True, indent=4, cls=MessageJsonEncoder)
    click.echo(json_output)

Response: View the output of the datafeeds and notifications used in your DASHBOARDS


Instruction:
Can you generate a brief explanation for the following Python 3 code
def list(component_type):
    config_loader = initialise_component_loader()
    component_types = sorted({
         "displays": lambda: config_loader.load_by_type(ComponentType.DISPLAY),
         "datafeeds": lambda: config_loader.load_by_type(ComponentType.DATA_FEED),
         "filters": lambda: config_loader.load_by_type(ComponentType.FILTER),
         "notifications": lambda: config_loader.load_by_type(ComponentType.NOTIFICATION)
    }.items(), key=lambda t: t[0])

    def print_ids(creators):
        ids = {c.id_key_value[1] if hasattr(c, "id_key_value") else c.get_id() for c in creators}
        for i in sorted(ids):
            click.echo(" - %s" % i)

    for k, v in component_types:
        if component_type == k or component_type == "all":
            click.echo("Available %s:" % k)
            print_ids(v())

        if component_type == "all":
            click.echo("")

Response: List all available components of a given type


Instruction:
Write a Python 3 script to
parse the section of configuration pertaining to a component

Response:
def parse(self, config):
        """
            Parses the section of configuration pertaining to a component
            :param config: dict of specific config section
            :return:
        """

        if "type" not in config:
            raise InvalidConfigurationException("The dashboard configuration has not defined a 'type'. %s" % config)

        component_type = config["type"]
        component_config = self._get_config_by_id(self._component_configs, component_type)

        if not component_config:
            raise ComponentNotFoundForType(component_type)

        options = config.get("options", {})
        component = self._parse_item(component_config, options, config)
        component.name = config.get("name", "")
        return component


Instruction:
Can you tell what is the following Python 3 function doing
def err_msg(self, instance, value):
        if not hasattr(self, "name"):
            # err_msg will be called by the composed descriptor
            return ""
        return (
            "Attempted to set the {f_type} attribute {inst}.{attr} to the "
            "{val_type} value {val}, which does not satisfy the condition "
            "{f_type}.".format(
                f_type=self.field_type,
                inst=instance.__class__.__name__,
                attr=self.name,
                val_type=value.__class__.__name__,
                val=value))

Response:
Return an error message for use in exceptions thrown by
            subclasses.


Instruction:
Can you tell what is the following Python 3 function doing
def exc_thrown_by_descriptor():
        traceback = sys.exc_info()[2]
        tb_locals = traceback.tb_frame.f_locals
        # relying on naming convention to get the object that threw
        # the exception
        if "self" in tb_locals:
            if not isinstance(tb_locals["self"], Descriptor):
                return False
            return True
        return False

Response:
Return True if the last exception was thrown by a
            Descriptor instance.


Instruction:
Can you generate the documentation for the following Python 3 function
def _set_data(self):
        if getattr(self, 'data', False) and not getattr(self, '_x', False) and not getattr(self, '_y', False):
            _x = XVariable()
            _y = YVariable()
            _x.contribute_to_class(self, 'X', self.data)
            _y.contribute_to_class(self, 'Y', self.data)
            self['data'] = zip(self._x.points, self._y.points)
        else:
            for axis in ('_x', '_y'):
                axis_obj = getattr(self, axis, False)
                if not axis_obj:
                    raise exception.MissingAxisException("%s missing" % axis)
                if not getattr(axis_obj, 'points', False):
                    raise exception.MissingDataException()

            self['data'] = zip(self._x.points, self._y.points)

Response: This method will be called to set the data of the current object.


Instruction:
Write a Python 3 script for
setting the graph ploting options

Response:
def _set_options(self):
        "sets the graph ploting options"
        # this is aweful
        # FIXME: Axis options should be passed completly by a GraphOption
        if 'xaxis' in self._options.keys():
            self._options['xaxis'].update(
                        {'mode' : self._get_axis_mode(XAxis._var_name)})
        if 'yaxis' in self._options.keys():
            self._options['yaxis'].update(
                        {'mode' : self._get_axis_mode(YAxis._var_name)})


Instruction:
Implement a function in Python 3 to
create an __init__ method that sets all the attributes necessary for the Descriptor to check the value.

Response:
def create_init(attrs):
    """Create an __init__ method that sets all the attributes
    necessary for the function the Descriptor invokes to check the
    value.

    """
    args = ", ".join(attrs)
    vals = ", ".join(['getattr(self, "{}")'.format(attr) for attr in attrs])
    attr_lines = "\n    ".join(
        ["self.{attr} = {attr}".format(attr=attr) for attr in attrs])
    init_code = """def _init(self, {args}):
    super(self.__class__, self).__init__()
    {attr_lines}
    self.field_type += "({{}})".format(
        ", ".join([str(val) for val in [{vals}]]))
    """.format(args=args, attr_lines=attr_lines, vals=vals)
    exec(init_code, globals())
    return _init


Instruction:
Can you generate the documentation for the following Python 3 function
def create_setter(func, attrs):
    def _set(self, instance, value, name=None):
        args = [getattr(self, attr) for attr in attrs]
        if not func(value, *args):
            raise ValueError(self.err_msg(instance, value))
    return _set

Response: Create the __set__ method for the descriptor.


Instruction:
Write a Python 3 script to
turn a funcs list element into a class object.

Response:
def make_class(clsname, func, attrs):
    """Turn a funcs list element into a class object."""
    clsdict = {"__set__": create_setter(func, attrs)}
    if len(attrs) > 0:
        clsdict["__init__"] = create_init(attrs)
    clsobj = type(str(clsname), (Descriptor, ), clsdict)
    clsobj.__doc__ = docstrings.get(clsname)
    return clsobj


Instruction:
How would you explain what the following Python 3 function does
def cycle(self):
        messages = self.poll_datafeeds()
        notifications = self.process_notifications(messages)

        self.draw_notifications(notifications)

Response: Cycle through notifications with latest results from data feeds.


Instruction:
Implement a function in Python 3 to
convert value to a numeric value or raise a ValueError if that isn t possible.

Response:
def try_convert(value):
        """Convert value to a numeric value or raise a ValueError
        if that isn't possible.

        """
        convertible = ForceNumeric.is_convertible(value)
        if not convertible or isinstance(value, bool):
            raise ValueError
        if isinstance(str(value), str):
            return ForceNumeric.str_to_num(value)
        return float(value)


Instruction:
Can you implement a function in Python 3 that
converts a string value to an int or a float depending on the numeric value represented by str_value.

Response:
def str_to_num(str_value):
        """Convert str_value to an int or a float, depending on the
        numeric value represented by str_value.

        """
        str_value = str(str_value)
        try:
            return int(str_value)
        except ValueError:
            return float(str_value)


Instruction:
Write a Python 3 script for
plotting a single node into a template.

Response:
def plot(parser, token):
    """
    Tag to plot graphs into the template
    """

    tokens = token.split_contents()
    tokens.pop(0)
    graph = tokens.pop(0)

    attrs = dict([token.split("=") for token in tokens])

    if 'id' not in attrs.keys():
        attrs['id'] = ''.join([chr(choice(range(65, 90))) for i in range(0, 5)])
    else:
        attrs['id'] = attrs['id'][1:len(attrs['id'])-1]

    attr_string = ''.join([" %s=%s" % (k, v) for k, v in attrs.iteritems()])
    return GraphRenderer(graph, attr_string, attrs['id'])


Instruction:
Write a Python 3 function that can
try to get a Unicode copy of a string.

Response:
def force_unicode(raw):
    '''Try really really hard to get a Unicode copy of a string.

    First try :class:`BeautifulSoup.UnicodeDammit` to try to force
    to Unicode; if that fails, assume UTF-8 encoding, and ignore
    all errors.

    :param str raw: string to coerce
    :return: Unicode approximation of `raw`
    :returntype: :class:`unicode`

    '''
    converted = UnicodeDammit(raw, isHTML=True)
    if not converted.unicode:
        converted.unicode = unicode(raw, 'utf8', errors='ignore')

    encoding_m = encoding_re.match(converted.unicode)
    if encoding_m:
        converted.unicode = \
            encoding_m.group('start_xml') + \
            encoding_m.group('remainder')

    return converted.unicode


Instruction:
Can you generate a brief explanation for the following Python 3 code
def make_clean_html(raw, stream_item=None, encoding=None):
    '''Get a clean text representation of presumed HTML.

    Treat `raw` as though it is HTML, even if we have no idea what it
    really is, and attempt to get a properly formatted HTML document
    with all HTML-escaped characters converted to their unicode.

    This is called below by the `clean_html` transform stage, which
    interprets MIME-type.  If `character_encoding` is not provided,
    and `stream_item` is provided, then this falles back to
    :attr:`streamcorpus.StreamItem.body.encoding`.

    :param str raw: raw text to clean up
    :param stream_item: optional stream item with encoding metadata
    :type stream_item: :class:`streamcorpus.StreamItem`
    :returns: UTF-8-encoded byte string of cleaned HTML text
    :returntype: :class:`str`

    '''
    # Fix emails by protecting the <,> from HTML
    raw = fix_emails(raw)
    raw_decoded = nice_decode(raw, stream_item=stream_item, encoding=encoding)
    if raw_decoded is None:
        # give up on decoding it... maybe this should use force_unicode
        raw_decoded = raw

    # default attempt uses vanilla lxml.html
    try:
        root = lxml.html.document_fromstring(raw_decoded)
    except ValueError, exc:
        if 'with encoding declaration' in str(exc):
            root = lxml.html.document_fromstring(raw)
        else:
            raise

    # While we have the document parsed as a DOM, let's strip attributes.
    # (The HTML cleaner seems to only support whitelisting attributes.
    # As of now, we just want to blacklist a few.)
    lxml.etree.strip_attributes(root, 'class', 'id')

    # if that worked, then we will be able to generate a
    # valid HTML string
    fixed_html = lxml.html.tostring(root, encoding=unicode)

    # remove any ^M characters
    fixed_html = string.replace(fixed_html, '\r', ' ')

    # We drop utf8 characters that are above 0xFFFF as
    # Lingpipe seems to be doing the wrong thing with them.
    fixed_html = drop_invalid_and_upper_utf8_chars(fixed_html)

    # construct a Cleaner that removes any ``<script>`` tags,
    # Javascript, like an ``onclick`` attribute, comments, style
    # tags or attributes, ``<link>`` tags
    cleaner = lxml.html.clean.Cleaner(
        scripts=True, javascript=True,
        comments=True,
        # do not remove <html> <head> <title> etc
        page_structure=False,
        remove_tags=['base'],
        style=True, links=True)

    # now get the really sanitized HTML
    _clean_html = cleaner.clean_html(fixed_html)

    # generate pretty HTML in utf-8
    _clean_html = lxml.html.tostring(
        lxml.html.document_fromstring(_clean_html),
        method='html', encoding='utf-8',
        pretty_print=True,
        # include_meta_content_type=True
        )

    return uniform_html(_clean_html)

Response: Return a clean text representation of presumed HTML.


Instruction:
Can you tell what is the following Python 3 function doing
def uniform_html(html):
    '''Takes a utf-8-encoded string of HTML as input and returns a new
    HTML string with fixed quoting and close tags, which generally
    should not break any of the offsets and makes it easier for
    functions like
    :func:`streamcorpus_pipeline.offsets.char_offsets_to_xpaths` to
    operate without failures.

    '''
    doc = html5lib.parse(html.decode('utf-8'))
    config = {
        'omit_optional_tags': False,
        'encoding': 'utf-8',
        'quote_attr_values': 'always',
    }
    return html5lib.serializer.serialize(doc, **config)

Response:
Takes a utf - 8 - encoded string of HTML as input and returns a new
    HTML string with fixed quoting and close tags.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def is_matching_mime_type(self, mime_type):
        '''This implements the MIME-type matching logic for deciding whether
        to run `make_clean_html`

        '''
        if len(self.include_mime_types) == 0:
            return True
        if mime_type is None:
            return False
        mime_type = mime_type.lower()
        # NB: startswith is necessary here, because encodings are
        # often appended to HTTP header Content-Type
        return any(mime_type.startswith(mt) for mt in self.include_mime_types)

Response: This implements the MIME - type matching logic for deciding whether the content - type is matching the given MIME - type.


Instruction:
Implement a function in Python 3 to
extract a lower - case no - slashes domain name from a raw string that might be a URL

Response:
def domain_name_cleanse(raw_string):
    '''extract a lower-case, no-slashes domain name from a raw string
    that might be a URL
    '''
    try:
        parts = urlparse(raw_string)
        domain = parts.netloc.split(':')[0]
    except:
        domain = ''
    if not domain:
        domain = raw_string
    if not domain:
        return ''
    domain = re.sub('\/', '', domain.strip().lower())
    return domain


Instruction:
Make a summary of the following Python 3 code
def domain_name_left_cuts(domain):
    '''returns a list of strings created by splitting the domain on
    '.' and successively cutting off the left most portion
    '''
    cuts = []
    if domain:
        parts = domain.split('.')
        for i in range(len(parts)):
            cuts.append( '.'.join(parts[i:]))
    return cuts

Response:
returns a list of strings created by splitting the domain on
   . and successively cutting off the left most portion of the domain on
   .


Instruction:
How would you explain what the following Python 3 function does
def make_hash_kw(self, tok):
        '''Get a Murmur hash and a normalized token.

        `tok` may be a :class:`unicode` string or a UTF-8-encoded
        byte string.  :data:`DOCUMENT_HASH_KEY`, hash value 0, is
        reserved for the document count, and this function remaps
        that value.

        :param tok: token to hash
        :return: pair of normalized `tok` and its hash

        '''
        if isinstance(tok, unicode):
            tok = tok.encode('utf-8')
        h = mmh3.hash(tok)
        if h == DOCUMENT_HASH_KEY:
            h = DOCUMENT_HASH_KEY_REPLACEMENT
        return (tok, h)

Response: Get a Murmur hash and a normalized token.


Instruction:
How would you implement a function in Python 3 that
collects all of the words to be indexed from a stream item. This scans a stream item for all of the configured tagger IDs and returns a Counter of all of the words to be indexed.

Response:
def collect_words(self, si):
        '''Collect all of the words to be indexed from a stream item.

        This scans `si` for all of the configured tagger IDs.  It
        collects all of the token values (the
        :attr:`streamcorpus.Token.token`) and returns a
        :class:`collections.Counter` of them.

        :param si: stream item to scan
        :type si: :class:`streamcorpus.StreamItem`
        :return: counter of :class:`unicode` words to index
        :returntype: :class:`collections.Counter`

        '''
        counter = Counter()
        for tagger_id, sentences in si.body.sentences.iteritems():
            if ((self.keyword_tagger_ids is not None
                 and tagger_id not in self.keyword_tagger_ids)):
                continue
            for sentence in sentences:
                for token in sentence.tokens:
                    term = token.token  # always a UTF-8 byte string
                    term = term.decode('utf-8')
                    term = cleanse(term)
                    if ((self.keyword_size_limit is not None and
                         len(term) > self.keyword_size_limit)):
                        continue
                    if term not in self.stop_words:
                        counter[term] += 1
        return counter


Instruction:
Given the following Python 3 function, write the documentation
def index(self, si):
        '''Record index records for a single document.

        Which indexes this creates depends on the parameters to the
        constructor.  This records all of the requested indexes for
        a single document.

        '''
        if not si.body.clean_visible:
            logger.warn('stream item %s has no clean_visible part, '
                        'skipping keyword indexing', si.stream_id)
            return

        # Count tokens in si.clean_visible
        # We will recycle hash==0 for "# of documents"
        hash_counts = defaultdict(int)
        hash_counts[DOCUMENT_HASH_KEY] = 1
        hash_kw = defaultdict(int)
        words = self.collect_words(si)
        for tok, count in words.iteritems():
            (tok, tok_hash) = self.make_hash_kw(tok)
            hash_counts[tok_hash] += count
            hash_kw[tok] = tok_hash

        # Convert this and write it out
        if self.hash_docs:
            (k1, k2) = key_for_stream_item(si)
            kvps = [((h, k1, k2), n) for (h, n) in hash_counts.iteritems()
                    if h != DOCUMENT_HASH_KEY]
            self.client.put(HASH_TF_INDEX_TABLE, *kvps)

        if self.hash_frequencies:
            kvps = [((h,), 1) for h in hash_counts.iterkeys()]
            self.client.increment(HASH_FREQUENCY_TABLE, *kvps)

        if self.hash_keywords:
            kvps = [((h, t), 1) for (t, h) in hash_kw.iteritems()]
            self.client.increment(HASH_KEYWORD_INDEX_TABLE, *kvps)

Response: Record index records for a single document.


Instruction:
Given the following Python 3 function, write the documentation
def invert_hash(self, tok_hash):
        '''Get strings that correspond to some hash.

        No string will correspond to :data:`DOCUMENT_HASH_KEY`; use
        :data:`DOCUMENT_HASH_KEY_REPLACEMENT` instead.

        :param int tok_hash: Murmur hash to query
        :return: list of :class:`unicode` strings

        '''
        return [tok_encoded.decode('utf8')
                for (_, tok_encoded) in
                self.client.scan_keys(HASH_KEYWORD_INDEX_TABLE,
                                      ((tok_hash,), (tok_hash,)))]

Response: Get strings that correspond to some hash.


Instruction:
Can you implement a function in Python 3 that
gets the document frequencies for a list of hashes.

Response:
def document_frequencies(self, hashes):
        '''Get document frequencies for a list of hashes.

        This will return all zeros unless the index was written with
        `hash_frequencies` set.  If :data:`DOCUMENT_HASH_KEY` is
        included in `hashes`, that value will be returned with the
        total number of documents indexed.  If you are looking for
        documents with that hash, pass
        :data:`DOCUMENT_HASH_KEY_REPLACEMENT` instead.

        :param hashes: hashes to query
        :paramtype hashes: list of :class:`int`
        :return: map from hash to document frequency

        '''
        result = {}
        for (k, v) in self.client.get(HASH_FREQUENCY_TABLE,
                                      *[(h,) for h in hashes]):
            if v is None:
                v = 0
            result[k[0]] = v
        return result


Instruction:
Can you generate the documentation for the following Python 3 function
def lookup(self, h):
        '''Get stream IDs for a single hash.

        This yields strings that can be retrieved using
        :func:`streamcorpus_pipeline._kvlayer.get_kvlayer_stream_item`,
        or fed back into :mod:`coordinate` or other job queue systems.

        Note that for common terms this can return a large number of
        stream IDs!  This is a scan over a dense region of a
        :mod:`kvlayer` table so it should be reasonably efficient,
        but be prepared for it to return many documents in a large
        corpus.  Blindly storing the results in a :class:`list`
        may be inadvisable.

        This will return nothing unless the index was written with
        :attr:`hash_docs` set.  No document will correspond to
        :data:`DOCUMENT_HASH_KEY`; use
        :data:`DOCUMENT_HASH_KEY_REPLACEMENT` instead.

        :param int h: Murmur hash to look up

        '''
        for (_, k1, k2) in self.client.scan_keys(HASH_TF_INDEX_TABLE,
                                                 ((h,), (h,))):
            yield kvlayer_key_to_stream_id((k1, k2))

Response: Get stream IDs for a single hash.


Instruction:
Here you have a function in Python 3, explain what it does
def lookup_tf(self, h):
        '''Get stream IDs and term frequencies for a single hash.

        This yields pairs of strings that can be retrieved using
        :func:`streamcorpus_pipeline._kvlayer.get_kvlayer_stream_item`
        and the corresponding term frequency.

        ..see:: :meth:`lookup`

        '''
        for ((_, k1, k2), v) in self.client.scan(HASH_TF_INDEX_TABLE,
                                                 ((h,), (h,))):
            yield (kvlayer_key_to_stream_id((k1, k2)), v)

Response: Get stream IDs and term frequencies for a single hash.


Instruction:
Write a Python 3 script for
giving a spinn3r feed produce a sequence of valid StreamItems.

Response:
def _make_stream_items(f):
    """Given a spinn3r feed, produce a sequence of valid StreamItems.

    Because of goopy Python interactions, you probably need to call
    this and re-yield its results, as

    >>> with open(filename, 'rb') as f:
    ...   for si in _make_stream_items(f):
    ...     yield si

    """
    reader = ProtoStreamReader(f)
    return itertools.ifilter(
        lambda x: x is not None,
        itertools.imap(_make_stream_item, reader))


Instruction:
Can you implement a function in Python 3 that
gives a single spinn3r feed entry produce a single StreamItem. Returns None if a complete item can t be constructed.

Response:
def _make_stream_item(entry):
    """Given a single spinn3r feed entry, produce a single StreamItem.

    Returns 'None' if a complete item can't be constructed.

    """
    # get standard metadata, assuming it's present...
    if not hasattr(entry, 'permalink_entry'):
        return None
    pe = entry.permalink_entry

    # ...and create a streamitem...
    si = streamcorpus.make_stream_item(
        pe.date_found[:-1] + '.0Z',
        pe.canonical_link.href.encode('utf8'))
    if not si.stream_time:
        logger.debug('failed to generate stream_time from {0!r}'
                     .format(pe.date_found))
        return None
    if not si.abs_url:
        logger.debug('failed to generate abs_url from {0!r}'
                     .format(pe.canonical_link.href))
        return None

    # ...filling in the actual data
    si.body = _make_content_item(
        pe.content,
        alternate_data=entry.feed_entry.content.data)
    if not si.body:
        return None
    if not si.body.raw:
        return None

    if pe.content_extract.data:
        si.other_content['extract'] = _make_content_item(pe.content_extract)
    si.other_content['title'] = streamcorpus.ContentItem(
        raw=pe.title.encode('utf8'),
        media_type=pe.content_extract.mime_type,
        encoding='UTF-8')
    si.other_content['feed_entry_title'] = streamcorpus.ContentItem(
        raw=entry.feed_entry.title.encode('utf8'),
        media_type=entry.feed_entry.content.mime_type,
        encoding='UTF-8')
    if entry.feed_entry.content.data:
        si.other_content['feed_entry'] = _make_content_item(
            entry.feed_entry.content)
    si.source_metadata['lang'] = pe.lang[0].code
    si.source_metadata['author'] = json.dumps(
        dict(
            name=pe.author[0].name,
            email=pe.author[0].email,
            link=pe.author[0].link[0].href,
        )
    )
    si.source = entry.source.publisher_type
    return si


Instruction:
Write a Python 3 function that can
create a ContentItem from a node in the spinn3r protobuf data tree.

Response:
def _make_content_item(node, mime_type=None, alternate_data=None):
    """Create a ContentItem from a node in the spinn3r data tree.

    The ContentItem is created with raw data set to ``node.data``,
    decompressed if the node's encoding is 'zlib', and UTF-8
    normalized, with a MIME type from ``node.mime_type``.

    ``node``
      the actual node from the spinn3r protobuf data
    ``mime_type``
      string MIME type to use (defaults to ``node.mime_type``)
    ``alternate_data``
      alternate (compressed) data to use, if ``node.data`` is missing
      or can't be decompressed

    """
    raw = node.data
    if getattr(node, 'encoding', None) == 'zlib':
        try:
            raw = zlib.decompress(node.data)
        except Exception, exc:
            if alternate_data is not None:
                try:
                    raw = zlib.decompress(alternate_data)
                except Exception:
                    raise exc  # the original exception
                else:
                    raise
    if mime_type is None:
        mime_type = node.mime_type
    raw = raw.decode('utf8').encode('utf8')
    return streamcorpus.ContentItem(raw=raw, media_type=mime_type)


Instruction:
Here you have a function in Python 3, explain what it does
def _read(self, n):
        if n <= len(self._prefix):
            # the read can be fulfilled entirely from the prefix
            result = self._prefix[:n]
            self._prefix = self._prefix[n:]
            return result
        # otherwise we need to read some
        n -= len(self._prefix)
        result = self._prefix + self.f.read(n)
        self._prefix = ""
        return result

Response: Read up to n bytes from the underlying file.


Instruction:
Can you write a function in Python 3 where it
reads exactly a varint out of the underlying file.

Response:
def _read_varint(self):
        """Read exactly a varint out of the underlying file."""
        buf = self._read(8)
        (n, l) = _DecodeVarint(buf, 0)
        self._unread(buf[l:])
        return n


Instruction:
Given the following Python 3 function, write the documentation
def _read_a(self, cls):
        o = cls()
        o.ParseFromString(self._read_block())
        return o

Response: Read some protobuf - encoded object out of the file.


Instruction:
Here you have a function in Python 3, explain what it does
def parse_keys_and_ranges(i_str, keyfunc, rangefunc):
    '''Parse the :class:`from_kvlayer` input string.

    This accepts two formats.  In the textual format, it accepts any
    number of stream IDs in timestamp-docid format, separated by ``,``
    or ``;``, and processes those as individual stream IDs.  In the
    binary format, it accepts 20-byte key blobs (16 bytes md5 hash, 4
    bytes timestamp) split by ``;`` or ``<``; e.g., ``a<f;x`` loads
    scans keys `a` through `f` and loads singly key `x`.

    `keyfunc` and `rangefunc` are run as generators and their yields
    are yielded from this function.

    '''
    while i_str:
        m = _STREAM_ID_RE.match(i_str)
        if m:
            # old style text stream_id
            for retval in keyfunc(stream_id_to_kvlayer_key(m.group())):
                yield retval
            i_str = i_str[m.end():]
            while i_str and ((i_str[0] == ',') or (i_str[0] == ';')):
                i_str = i_str[1:]
            continue

        if len(i_str) == SI_KEY_LENGTH:
            # one key, get it.
            key = parse_si_key(i_str)
            for retval in keyfunc(key):
                yield retval
            return

        keya = i_str[:SI_KEY_LENGTH]
        splitc = i_str[SI_KEY_LENGTH]
        if splitc == '<':
            # range
            keyb = i_str[SI_KEY_LENGTH+1:SI_KEY_LENGTH+1+SI_KEY_LENGTH]
            i_str = i_str[SI_KEY_LENGTH+1+SI_KEY_LENGTH:]
            keya = parse_si_key(keya)
            keyb = parse_si_key(keyb)
            for retval in rangefunc(keya, keyb):
                yield retval
        elif splitc == ';':
            # keya is single key to load
            keya = parse_si_key(keya)
            for retval in keyfunc(keya):
                yield retval
            i_str = i_str[SI_KEY_LENGTH+1+1:]
        else:
            logger.error('bogus key splitter %s, %r', splitc, i_str)
            return

Response: Parse the : class from_kvlayer input string.


Instruction:
How would you code a function in Python 3 to
retrieve a : class : streamcorpus. StreamItem from the kvlayer database.

Response:
def get_kvlayer_stream_item(client, stream_id):
    '''Retrieve a :class:`streamcorpus.StreamItem` from :mod:`kvlayer`.

    This function requires that `client` already be set up properly::

        client = kvlayer.client()
        client.setup_namespace(STREAM_ITEM_TABLE_DEFS,
                               STREAM_ITEM_VALUE_DEFS)
        si = get_kvlayer_stream_item(client, stream_id)

    `stream_id` is in the form of
    :data:`streamcorpus.StreamItem.stream_id` and contains the
    ``epoch_ticks``, a hyphen, and the ``doc_id``.

    :param client: kvlayer client object
    :type client: :class:`kvlayer.AbstractStorage`
    :param str stream_id: stream Id to retrieve
    :return: corresponding :class:`streamcorpus.StreamItem`
    :raise exceptions.KeyError: if `stream_id` is malformed or does
      not correspond to anything in the database

    '''
    if client is None:
        client = kvlayer.client()
        client.setup_namespace(STREAM_ITEM_TABLE_DEFS,
                               STREAM_ITEM_VALUE_DEFS)
    key = stream_id_to_kvlayer_key(stream_id)
    for k, v in client.get(STREAM_ITEMS_TABLE, key):
        if v is not None:
            errors, bytestr = streamcorpus.decrypt_and_uncompress(v)
            return streamcorpus.deserialize(bytestr)
    raise KeyError(stream_id)


Instruction:
Given the following Python 3 function, write the documentation
def make_doc_id_range(doc_id):
    '''Construct a tuple(begin, end) of one-tuple kvlayer keys from a
    hexdigest doc_id.

    '''
    assert len(doc_id) == 32, 'expecting 32 hex string, not: %r' % doc_id
    bin_docid = base64.b16decode(doc_id.upper())
    doc_id_range = ((bin_docid,), (bin_docid,))
    return doc_id_range

Response:
Construct a tuple of begin end of one - tuple kvlayer keys from a
    hexdigest doc_id.


Instruction:
Here you have a function in Python 3, explain what it does
def get_kvlayer_stream_item_by_doc_id(client, doc_id):
    '''Retrieve :class:`streamcorpus.StreamItem`s from :mod:`kvlayer`.

    Namely, it returns an iterator over all documents with the given
    docid. The docid should be an md5 hash of the document's abs_url.

    :param client: kvlayer client object
    :type client: :class:`kvlayer.AbstractStorage`
    :param str doc_id: doc id of documents to retrieve
    :return: generator of :class:`streamcorpus.StreamItem`
    '''
    if client is None:
        client = kvlayer.client()
        client.setup_namespace(STREAM_ITEM_TABLE_DEFS,
                               STREAM_ITEM_VALUE_DEFS)
    doc_id_range = make_doc_id_range(doc_id)
    for k, v in client.scan(STREAM_ITEMS_TABLE, doc_id_range):
        if v is not None:
            errors, bytestr = streamcorpus.decrypt_and_uncompress(v)
            yield streamcorpus.deserialize(bytestr)

Response: Retrieve all stream items with the given doc_id from the given kvlayer.


Instruction:
Implement a Python 3 function for
retrieving stream ids from kvlayer.

Response:
def get_kvlayer_stream_ids_by_doc_id(client, doc_id):
    '''Retrieve stream ids from :mod:`kvlayer`.

    Namely, it returns an iterator over all stream ids with the given
    docid. The docid should be an md5 hash of the document's abs_url.

    :param client: kvlayer client object
    :type client: :class:`kvlayer.AbstractStorage`
    :param str doc_id: doc id of documents to retrieve
    :return: generator of str
    '''
    if client is None:
        client = kvlayer.client()
        client.setup_namespace(STREAM_ITEM_TABLE_DEFS,
                               STREAM_ITEM_VALUE_DEFS)
    doc_id_range = make_doc_id_range(doc_id)
    for k in client.scan_keys(STREAM_ITEMS_TABLE, doc_id_range):
        yield kvlayer_key_to_stream_id(k)


Instruction:
Write a Python 3 function that can
serialize a StreamItem kvlayer key.

Response:
def serialize_si_key(si_key):
    '''
    Return packed bytes representation of StreamItem kvlayer key.
    The result is 20 bytes, 16 of md5 hash, 4 of int timestamp.
    '''
    if len(si_key[0]) != 16:
        raise ValueError('bad StreamItem key, expected 16 byte '
                         'md5 hash binary digest, got: {0!r}'.format(si_key))
    return struct.pack('>16si', si_key[0], si_key[1])


Instruction:
Given the following Python 3 function, write the documentation
def streamitem_to_key_data(si):
    '''
    extract the parts of a StreamItem that go into a kvlayer key,
    convert StreamItem to blob for storage.

    return (kvlayer key tuple), data blob
    '''
    key = key_for_stream_item(si)
    data = streamcorpus.serialize(si)
    errors, data = streamcorpus.compress_and_encrypt(data)
    assert not errors, errors
    return key, data

Response: Convert a StreamItem into a kvlayer key tuple and data blob for storage.


Instruction:
Write a Python 3 function that can
change the working directory and restore the previous on exit

Response:
def working_directory(path):
    """Change working directory and restore the previous on exit"""
    prev_dir = os.getcwd()
    os.chdir(str(path))
    try:
        yield
    finally:
        os.chdir(prev_dir)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def strip_prefix(s, prefix, strict=False):
    if s.startswith(prefix):
        return s[len(prefix) :]
    elif strict:
        raise WimpyError("string doesn't start with prefix")
    return s

Response: Removes the prefix from the input string if it s there otherwise returns input string unchanged.


Instruction:
Implement a Python 3 function for
removing the suffix from the input string if it s present otherwise returns input string unchanged.

Response:
def strip_suffix(s, suffix, strict=False):
    """Removes the suffix, if it's there, otherwise returns input string unchanged.
    If strict is True, also ensures the suffix was present"""
    if s.endswith(suffix):
        return s[: len(s) - len(suffix)]
    elif strict:
        raise WimpyError("string doesn't end with suffix")
    return s


Instruction:
Write a Python 3 script to
am the element of needle contained in haystack?

Response:
def is_subsequence(needle, haystack):
    """Are all the elements of needle contained in haystack, and in the same order?
    There may be other elements interspersed throughout"""
    it = iter(haystack)
    for element in needle:
        if element not in it:
            return False
    return True


Instruction:
Write a Python 3 function that can
create a new Ice application with a default home page.

Response:
def cube():
    """Return an Ice application with a default home page.

    Create :class:`Ice` object, add a route to return the default page
    when a client requests the server root, i.e. /, using HTTP GET
    method, add an error handler to return HTTP error pages when an
    error occurs and return this object. The returned object can be used
    as a WSGI application.

    Returns:
      Ice: WSGI application.
    """
    app = Ice()

    @app.get('/')
    def default_home_page():
        """Return a default home page."""
        return simple_html('It works!',
                           '<h1>It works!</h1>\n'
                           '<p>This is the default ice web page.</p>')

    @app.error()
    def generic_error_page():
        """Return a simple and generic error page."""
        return simple_html(app.response.status_line,
                           '<h1>{title}</h1>\n'
                           '<p>{description}</p>\n'
                           '<hr>\n'
                           '<address>Ice/{version}</address>'.format(
                           title=app.response.status_line,
                           description=app.response.status_detail,
                           version=__version__))

    def simple_html(title, body):
        """Return a simple HTML page."""
        return (
            '<!DOCTYPE html>\n'
            '<html>\n<head><title>{title}</title></head>\n'
            '<body>\n{body}\n</body>\n</html>\n'
        ).format(title=title, body=body)

    return app


Instruction:
Can you tell what is the following Python 3 function doing
def run(self, host='127.0.0.1', port=8080):
        from wsgiref import simple_server
        self._server = simple_server.make_server(host, port, self)
        self._server.serve_forever()

Response: Run the application using a simple WSGI server.


Instruction:
Explain what the following Python 3 code does
def exit(self):
        if self._server is not None:
            self._server.shutdown()
            self._server.server_close()
            self._server = None

Response: Stop the simple WSGI server running the appliation.


Instruction:
Make a summary of the following Python 3 code
def route(self, method, pattern):
        def decorator(callback):
            self._router.add(method, pattern, callback)
            return callback
        return decorator

Response: Decorator to add a route for a request with any HTTP method.


Instruction:
Write a Python 3 script for
sending content of a static file as response.

Response:
def static(self, root, path, media_type=None, charset='UTF-8'):
        """Send content of a static file as response.

        The path to the document root directory should be specified as
        the root argument. This is very important to prevent directory
        traversal attack. This method guarantees that only files within
        the document root directory are served and no files outside this
        directory can be accessed by a client.

        The path to the actual file to be returned should be specified
        as the path argument. This path must be relative to the document
        directory.

        The *media_type* and *charset* arguments are used to set the
        Content-Type header of the HTTP response. If *media_type*
        is not specified or specified as ``None`` (the default), then it
        is guessed from the filename of the file to be returned.

        Arguments:
          root (str): Path to document root directory.
          path (str): Path to file relative to document root directory.
          media_type (str, optional): Media type of file.
          charset (str, optional): Character set of file.

        Returns:
          bytes: Content of file to be returned in the HTTP response.
        """
        root = os.path.abspath(os.path.join(root, ''))
        path = os.path.abspath(os.path.join(root, path.lstrip('/\\')))

        # Save the filename from the path in the response state, so that
        # a following download() call can default to this filename for
        # downloadable file when filename is not explicitly specified.
        self.response.state['filename'] = os.path.basename(path)

        if not path.startswith(root):
            return 403
        elif not os.path.isfile(path):
            return 404

        if media_type is not None:
            self.response.media_type = media_type
        else:
            self.response.media_type = mimetypes.guess_type(path)[0]
        self.response.charset = charset

        with open(path, 'rb') as f:
            return f.read()


Instruction:
How would you explain what the following Python 3 function does
def download(self, content, filename=None,
                 media_type=None, charset='UTF-8'):
        """Send content as attachment (downloadable file).

        The *content* is sent after setting Content-Disposition header
        such that the client prompts the user to save the content
        locally as a file. An HTTP response status code may be specified
        as *content*. If the status code is not ``200``, then this
        method does nothing and returns the status code.

        The filename used for the download is determined according to
        the following rules. The rules are followed in the specified
        order.

          1. If *filename* is specified, then the base name from this
             argument, i.e. ``os.path.basename(filename)``, is used as the
             filename for the download.
          2. If *filename* is not specified or specified as ``None``
             (the default), then the base name from the file path
             specified to a previous :meth:`static` call made while
             handling the current request is used.
          3. If *filename* is not specified and there was no
             :meth:`static` call made previously for the current
             request, then the base name from the current HTTP request
             path is used.
          4. As a result of the above steps, if the resultant *filename*
             turns out to be empty, then :exc:`ice.LogicError` is raised.

        The *media_type* and *charset* arguments are used in the same
        manner as they are used in :meth:`static`.

        Arguments:
          content (str, bytes or int): Content to be sent as download or
            HTTP status code of the response to be returned.
          filename (str): Filename to use for saving the content
          media_type (str, optional): Media type of file.
          charset (str, optional): Character set of file.

        Returns:
          content, i.e. the first argument passed to this method.

        Raises:
          LogicError: When filename cannot be determined.
        """
        if isinstance(content, int) and content != 200:
            return content
        if filename is not None:
            filename = os.path.basename(filename)
        elif 'filename' in self.response.state:
            filename = self.response.state['filename']
        else:
            filename = os.path.basename(self.request.path)

        if filename == '':
            raise LogicError('Cannot determine filename for download')

        if media_type is not None:
            self.response.media_type = media_type
        else:
            self.response.media_type = mimetypes.guess_type(filename)[0]
        self.response.charset = charset
        self.response.add_header('Content-Disposition', 'attachment; '
                                 'filename="{}"'.format(filename))
        return content

Response: Send content as attachment.


Instruction:
Given the following Python 3 function, write the documentation
def _get_error_page_callback(self):
        if self.response.status in self._error_handlers:
            return self._error_handlers[self.response.status]
        elif None in self._error_handlers:
            return self._error_handlers[None]
        else:
            # Rudimentary error handler if no error handler was found
            self.response.media_type = 'text/plain'
            return lambda: self.response.status_line

Response: Return an error page callback for the current response status.


Instruction:
Can you create a Python 3 function that
adds a route to the routing table.

Response:
def add(self, method, pattern, callback):
        """Add a route.

        Arguments:
          method (str): HTTP method, e.g. GET, POST, etc.
          pattern (str): Pattern that request paths must match.
          callback (str): Route handler that is invoked when a request
            path matches the *pattern*.
        """
        pat_type, pat = self._normalize_pattern(pattern)
        if pat_type == 'literal':
            self._literal[method][pat] = callback
        elif pat_type == 'wildcard':
            self._wildcard[method].append(WildcardRoute(pat, callback))
        else:
            self._regex[method].append(RegexRoute(pat, callback))


Instruction:
Write a Python 3 script for
checking if there is at least one handler for method.

Response:
def contains_method(self, method):
        """Check if there is at least one handler for *method*.

        Arguments:
          method (str): HTTP method name, e.g. GET, POST, etc.

        Returns:
          ``True`` if there is at least one route defined for *method*,
          ``False`` otherwise
        """
        return method in itertools.chain(self._literal, self._wildcard,
                                         self._regex)


Instruction:
How would you implement a function in Python 3 that
resolves a request to a route handler.

Response:
def resolve(self, method, path):
        """Resolve a request to a route handler.

        Arguments:
          method (str): HTTP method, e.g. GET, POST, etc. (type: str)
          path (str): Request path

        Returns:
          tuple or None: A tuple of three items:

            1. Route handler (callable)
            2. Positional arguments (list)
            3. Keyword arguments (dict)

          ``None`` if no route matches the request.
        """
        if method in self._literal and path in self._literal[method]:
            return self._literal[method][path], [], {}
        else:
            return self._resolve_non_literal_route(method, path)


Instruction:
Write a Python 3 script to
resolve a request to a wildcard or regex route handler.

Response:
def _resolve_non_literal_route(self, method, path):
        """Resolve a request to a wildcard or regex route handler.

        Arguments:
          method (str): HTTP method name, e.g. GET, POST, etc.
          path (str): Request path

        Returns:
          tuple or None: A tuple of three items:

            1. Route handler (callable)
            2. Positional arguments (list)
            3. Keyword arguments (dict)

          ``None`` if no route matches the request.
        """
        for route_dict in (self._wildcard, self._regex):
            if method in route_dict:
                for route in reversed(route_dict[method]):
                    callback_data = route.match(path)
                    if callback_data is not None:
                        return callback_data
        return None


Instruction:
Can you create a Python 3 function that
returns a normalized form of the pattern.

Response:
def _normalize_pattern(pattern):
        """Return a normalized form of the pattern.

        Normalize the pattern by removing pattern type prefix if it
        exists in the pattern. Then return the pattern type and the
        pattern as a tuple of two strings.

        Arguments:
          pattern (str): Route pattern to match request paths

        Returns:
          tuple: Ruple of pattern type (str) and pattern (str)
        """
        if pattern.startswith('regex:'):
            pattern_type = 'regex'
            pattern = pattern[len('regex:'):]
        elif pattern.startswith('wildcard:'):
            pattern_type = 'wildcard'
            pattern = pattern[len('wildcard:'):]
        elif pattern.startswith('literal:'):
            pattern_type = 'literal'
            pattern = pattern[len('literal:'):]
        elif RegexRoute.like(pattern):
            pattern_type = 'regex'
        elif WildcardRoute.like(pattern):
            pattern_type = 'wildcard'
        else:
            pattern_type = 'literal'
        return pattern_type, pattern


Instruction:
Can you tell what is the following Python 3 function doing
def match(self, path):
        match = self._re.search(path)
        if match is None:
            return None
        args = []
        kwargs = {}
        for i, wildcard in enumerate(self._wildcards):
            if wildcard.name == '!':
                continue
            value = wildcard.value(match.groups()[i])
            if not wildcard.name:
                args.append(value)
            else:
                kwargs[wildcard.name] = value
        return self._callback, args, kwargs

Response: Return route handler with arguments if path matches this route.


Instruction:
Write a Python 3 function for
returning route handler with arguments if path matches this route.

Response:
def match(self, path):
        """Return route handler with arguments if path matches this route.

        Arguments:
          path (str): Request path

        Returns:
          tuple or None: A tuple of three items:

            1. Route handler (callable)
            2. Positional arguments (list)
            3. Keyword arguments (dict)

          ``None`` if the route does not match the path.
        """
        match = self._re.search(path)
        if match is None:
            return None
        kwargs_indexes = match.re.groupindex.values()
        args_indexes = [i for i in range(1, match.re.groups + 1)
                          if i not in kwargs_indexes]
        args = [match.group(i) for i in args_indexes]
        kwargs = {}
        for name, index in match.re.groupindex.items():
            kwargs[name] = match.group(index)
        return self._callback, args, kwargs


Instruction:
Can you tell what is the following Python 3 function doing
def response(self):
        if isinstance(self.body, bytes):
            out = self.body
        elif isinstance(self.body, str):
            out = self.body.encode(self.charset)
        else:
            out = b''
        self.add_header('Content-Type', self.content_type)
        self.add_header('Content-Length', str(len(out)))

        self.start(self.status_line, self._headers)
        return [out]

Response: Return the HTTP response body as a sequence of bytes.


Instruction:
Given the following Python 3 function, write the documentation
def add_header(self, name, value):
        if value is not None:
            self._headers.append((name, value))

Response: Add an HTTP header to the response object.


Instruction:
Create a Python 3 function to
add a Set - Cookie header to the response object.

Response:
def set_cookie(self, name, value, attrs={}):
        """Add a Set-Cookie header to response object.

        For a description about cookie attribute values, see
        https://docs.python.org/3/library/http.cookies.html#http.cookies.Morsel.

        Arguments:
          name (str): Name of the cookie
          value (str): Value of the cookie
          attrs (dict): Dicitionary with cookie attribute keys and
                        values.
        """
        cookie = http.cookies.SimpleCookie()
        cookie[name] = value
        for key, value in attrs.items():
            cookie[name][key] = value
        self.add_header('Set-Cookie', cookie[name].OutputString())


Instruction:
Explain what the following Python 3 code does
def status_line(self):
        return (str(self.status) + ' ' +
                Response._responses[self.status].phrase)

Response: Return the HTTP status line.


Instruction:
Explain what the following Python 3 code does
def content_type(self):
        if (self.media_type is not None and
            self.media_type.startswith('text/') and
            self.charset is not None):
            return self.media_type + '; charset=' + self.charset
        else:
            return self.media_type

Response: Return the value of the Content - Type header field.


Instruction:
Write a Python 3 script for
returning the list of all values for the specified key.

Response:
def getall(self, key, default=[]):
        """Return the list of all values for the specified key.

        Arguments:
          key (object): Key
          default (list): Default value to return if the key does not
            exist, defaults to ``[]``, i.e. an empty list.

        Returns:
          list: List of all values for the specified key if the key
          exists, ``default`` otherwise.
        """
        return self.data[key] if key in self.data else default


Instruction:
Can you tell what is the following Python 3 function doing
def rmtree(path, use_shutil=True, followlinks=False, retries=10):
    '''remove all files and directories below path, including path
    itself; works even when shutil.rmtree fails because of read-only
    files in NFS and Windows.  Follows symlinks.

    `use_shutil` defaults to True; useful for testing

    `followlinks` defaults to False; if set to True, shutil.rmtree is
    not used.
    '''
    if use_shutil and not followlinks:
        try:
            shutil.rmtree(path)
            return
        except Exception, exc:
            logger.info('shutil.rmtree(%s) failed, so resorting to recursive delete', path)
            logger.debug('\ntrapped:\n%s', traceback.format_exc(exc))

    if not os.path.isdir(path):
        os.remove(path)
        return

    ## bottom up traversal removing files and then removing directories
    for root, dir_names, file_names in os.walk(path, topdown=False, followlinks=followlinks):
        for fname in file_names:
            fpath = os.path.join(root, fname)
            tries = 0
            while tries < retries:
                tries += 1
                try:
                    os.remove(fpath)
                    break
                except Exception, exc:
                    time.sleep(0.1)
            if os.path.exists(fpath):
                logger.critical('os.remove(%s) failed, so leaving data behind!!!', fpath)
                logger.critical('\ntrapped:\n%s', traceback.format_exc(exc))
                #logger.critical(get_open_fds())
        for dname in dir_names:
            full_path = os.path.join(root, dname)
            if os.path.islink(full_path):
                real_path = os.path.realpath(full_path)
                os.remove(full_path)
                full_path = real_path
            os.rmdir(full_path)

    if os.path.exists(path):
        os.rmdir(path)

Response: Recursively delete all files and directories below path.


Instruction:
Implement a Python 3 function for
getting a rejester. WorkUnit with KBA s3 path fetch it and save it some counts about it.

Response:
def rejester_run(work_unit):
     '''get a rejester.WorkUnit with KBA s3 path, fetch it, and save
     some counts about it.
     '''
     #fname = 'verify-chunks-%d-%d' % (os.getpid(), time.time())
     fname = work_unit.key.strip().split('/')[-1]

     output_dir_path = work_unit.data.get('output_dir_path', '/mnt')
     u = uuid.uuid3(uuid.UUID(int=0), work_unit.key.strip())
     path1 = u.hex[0]
     path2 = u.hex[1]
     fpath = os.path.join(output_dir_path, path1, path2, fname)
     if not os.path.exists(os.path.dirname(fpath)):
          os.makedirs(os.path.dirname(fpath))

     output = gzip.open(fpath + '-out.gz', 'wb')

     expected_si_count = int(fname.split('-')[1])

     max_tries = 20
     tries = 0
     while tries < max_tries:
          try:
               exc, si_count, serif_count, clean_visible_bytes, clean_visible_count, stream_ids = \
                   attempt_fetch(work_unit, fpath)
               if si_count != expected_si_count:
                    print 'retrying because si_count = %d != %d expected_si_count' % (si_count, expected_si_count)
                    sys.stdout.flush()
                    tries += 1
                    continue
               else:
                    print 'succeeded in reading si_count = %d' % (si_count,)
                    sys.stdout.flush()
               output.write( '%s\t%d\t%d\t%d\t%d\t%s\t%s\n' % (
                         exc, si_count, serif_count, clean_visible_bytes, clean_visible_count,
                         work_unit.key.strip(), ','.join(['%s|%s' % tup for tup in stream_ids])) )
               break
          except Exception, exc:
               print 'broken?'
               print traceback.format_exc(exc)
               sys.stdout.flush()
               tries += 1
               output.write(traceback.format_exc(exc))

     output.close()


Instruction:
Here you have a function in Python 3, explain what it does
def attempt_fetch(work_unit, fpath):
     '''attempt a fetch and iteration over a work_unit.key path in s3
     '''
     url = 'http://s3.amazonaws.com/aws-publicdatasets/' + work_unit.key.strip()

     ## cheapest way to iterate over the corpus is a few stages of
     ## streamed child processes.  Note that stderr needs to go
     ## separately to a file so that reading the stdin doesn't get
     ## blocked:
     cmd = '(wget -O - %s | gpg --no-permission-warning --trust-model always --output - --decrypt - | xz --decompress) 2> %s-err' % (url, fpath)
     print cmd
     child = Popen(cmd, stdout=PIPE, shell=True)
     print 'child launched'
     sys.stdout.flush()

     si_count = 0
     serif_count = 0
     exc = ''
     stream_ids = list()
     clean_visible_bytes = 0
     clean_visible_count = 0
     try:
         for si in Chunk(file_obj=child.stdout):
             print si.stream_id, si.abs_url
             if si.body.language:
                 lang = si.body.language.code
             else:
                 lang = ''
             stream_ids.append((lang, si.stream_id))
             if si.body.clean_visible:
                 clean_visible_count += 1
                 clean_visible_bytes += len(si.body.clean_visible)
             si_count += 1
             if 'serif' in si.body.sentences:
                 serif_count += 1
     except Exception, exc:
         exc = re.sub('\s+', ' ', str(exc)).strip()

     child.terminate()
     child.wait()
     child.stdout.close()
     return exc, si_count, serif_count, clean_visible_bytes, clean_visible_count, stream_ids

Response: attempt a fetch and iteration over a work_unit. key path in s3


Instruction:
Can you generate the documentation for the following Python 3 function
def get_file_lines(file_name):
    file_path = path.join(path.dirname(path.abspath(__file__)), file_name)
    with open(file_path) as file_obj:
        return [line for line in file_obj.read().splitlines() if line]

Response: Return a list of non - empty lines from file_path.


Instruction:
Explain what the following Python 3 code does
def get_describers():
    adjectives = map(lambda x: (x, 'prefix'), get_file_lines('adjectives.txt'))
    animal_nouns = map(lambda x: (x, 'suffix'), get_file_lines('nouns.txt'))
    return list(chain(adjectives, animal_nouns))

Response: Return a list of describer tuples in the form name position


Instruction:
Can you create a Python 3 function that
returns an ordered 2 - tuple containing a species and a describer.

Response:
def _random_adjspecies_pair():
    """Return an ordered 2-tuple containing a species and a describer."""
    describer, desc_position = random_describer()
    if desc_position == 'prefix':
        return (describer, random_species())
    elif desc_position == 'suffix':
        return (random_species(), describer)


Instruction:
Explain what the following Python 3 code does
def random_adjspecies_pair(maxlen=None, prevent_stutter=True):
    while True:
        pair = _random_adjspecies_pair()
        if maxlen and len(''.join(pair)) > maxlen:
            continue
        if prevent_stutter and pair[0][-1] == pair[1][0]:
            continue
        return pair

Response: Returns a random set of species and describer pairs.


Instruction:
Create a Python 3 function for
returning a random adjective or species separated by sep.

Response:
def random_adjspecies(sep='', maxlen=8, prevent_stutter=True):
    """
    Return a random adjective/species, separated by `sep`. The keyword
    arguments `maxlen` and `prevent_stutter` are the same as for
    `random_adjspecies_pair`, but note that the maximum length argument is
    not affected by the separator.
    """
    pair = random_adjspecies_pair(maxlen, prevent_stutter)
    return pair[0] + sep + pair[1]


Instruction:
Explain what the following Python 3 code does
def morph(ctx, app_id, sentence_file, json_flag,
          sentence, info_filter, pos_filter, request_id):
    # type: (Context, unicode, Optional[IO], bool, unicode, unicode, unicode, unicode) -> None  # NOQA
    """ Morphological analysis for Japanese."""

    app_id = clean_app_id(app_id)
    sentence = clean_sentence(sentence, sentence_file)

    if info_filter:
        info_filter = info_filter.replace(',', '|')

    if pos_filter:
        pos_filter = pos_filter.replace(',', '|')

    api = GoolabsAPI(app_id)
    ret = api.morph(
        sentence=sentence,
        info_filter=info_filter,
        pos_filter=pos_filter,
        request_id=request_id,
    )

    if json_flag:
        click.echo(format_json(api.response.json()))
        return

    for words in ret['word_list']:
        for word in words:
            click.echo(','.join(word))

Response: Morph the words in a sentence into a new node.


Instruction:
How would you implement a function in Python 3 that
gets similarity of two words.

Response:
def similarity(ctx, app_id, json_flag, query_pair, request_id):
    # type: (Context, unicode, bool, List[unicode], unicode) -> None
    """ Scoring the similarity of two words. """

    app_id = clean_app_id(app_id)

    api = GoolabsAPI(app_id)
    ret = api.similarity(
        query_pair=query_pair,
        request_id=request_id
    )

    if json_flag:
        click.echo(format_json(api.response.json()))
        return

    click.echo('{0:.16f}'.format(ret['score']))


Instruction:
Can you generate a brief explanation for the following Python 3 code
def hiragana(ctx, app_id, sentence_file,
             json_flag, sentence, output_type, request_id):
    # type: (Context, unicode, Optional[IO], bool, unicode, unicode, unicode) -> None # NOQA
    """ Convert the Japanese to Hiragana or Katakana. """

    app_id = clean_app_id(app_id)
    sentence = clean_sentence(sentence, sentence_file)

    api = GoolabsAPI(app_id)
    ret = api.hiragana(
        sentence=sentence,
        output_type=output_type,
        request_id=request_id
    )

    if json_flag:
        click.echo(format_json(api.response.json()))
        return

    click.echo(ret['converted'])

Response: Convert Japanese to Hiragana or Katakana.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def entity(ctx, app_id, sentence_file,
           json_flag, sentence, class_filter, request_id):
    # type: (Context, unicode, Optional[IO], bool, unicode, unicode, unicode) -> None # NOQA
    """ Extract unique representation from sentence. """

    app_id = clean_app_id(app_id)
    sentence = clean_sentence(sentence, sentence_file)

    if class_filter:
        class_filter = class_filter.replace(',', '|')

    api = GoolabsAPI(app_id)
    ret = api.entity(
        sentence=sentence,
        class_filter=class_filter,
        request_id=request_id
    )

    if json_flag:
        click.echo(format_json(api.response.json()))
        return

    for ne in ret['ne_list']:
        click.echo(','.join(ne))

Response: Extract unique representation of a sentence.


Instruction:
Given the following Python 3 function, write the documentation
def shortsum(ctx, app_id, review_file,
             json_flag, review, length, request_id):
    # type: (Context, unicode, Optional[IO], bool, unicode, unicode, unicode) -> None # NOQA
    """Summarize reviews into a short summary."""

    app_id = clean_app_id(app_id)
    review_list = clean_review(review, review_file)
    length_int = clean_length(length)  # type: Optional[int]

    api = GoolabsAPI(app_id)
    ret = api.shortsum(
        review_list=review_list,
        length=length_int,
        request_id=request_id,
    )

    if json_flag:
        click.echo(format_json(api.response.json()))
        return

    click.echo(ret['summary'])

Response: Summarize reviews into a short summary.


Instruction:
Here you have a function in Python 3, explain what it does
def keyword(ctx, app_id, body_file, json_flag,
            title, body, max_num, forcus, request_id):
    # type: (Context, unicode, Optional[IO], bool, unicode, unicode, int, unicode, unicode) -> None # NOQA
    """Extract "keywords" from an input document. """

    app_id = clean_app_id(app_id)
    body = clean_body(body, body_file)

    api = GoolabsAPI(app_id)
    ret = api.keyword(
        title=title,
        body=body,
        max_num=max_num,
        forcus=forcus,
        request_id=request_id,
    )

    if json_flag:
        click.echo(format_json(api.response.json()))
        return

    for k in ret['keywords']:
        k = dict((key.encode('utf-8'), k[key]) for key in k.keys())
        for keyword, score in six.iteritems(k):
            click.echo(u'{0},{1}'.format(text(keyword), score))

Response: Extract keywords from an input document.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def chrono(ctx, app_id, sentence_file,
           json_flag, sentence, doc_time, request_id):
    # type: (Context, unicode, Optional[IO], bool, unicode, unicode, unicode) -> None  # NOQA
    """Extract expression expressing date and time and normalize its value """

    app_id = clean_app_id(app_id)
    sentence = clean_sentence(sentence, sentence_file)

    api = GoolabsAPI(app_id)
    ret = api.chrono(
        sentence=sentence,
        doc_time=doc_time,
        request_id=request_id,
    )

    if json_flag:
        click.echo(format_json(api.response.json()))
        return

    for pair in ret['datetime_list']:
        click.echo(u'{0}: {1}'.format(text(pair[0]), pair[1]))

Response: Extract expression expressing date and time and normalize its value


Instruction:
Given the following Python 3 function, write the documentation
def create(self, stage, scp_config, config=None):
        '''Create a pipeline stage.

        Instantiates `stage` with `config`.  This essentially
        translates to ``stage(config)``, except that two keys from
        `scp_config` are injected into the configuration:
        ``tmp_dir_path`` is an execution-specific directory from
        combining the top-level ``tmp_dir_path`` configuration with
        :attr:`tmp_dir_suffix`; and ``third_dir_path`` is the same
        path from the top-level configuration.  `stage` may be either
        a callable returning the stage (e.g. its class), or its name
        in the configuration.

        `scp_config` is the configuration for the pipeline as a
        whole, and is required.  `config` is the configuration for
        the stage; if it is :const:`None` then it is extracted
        from `scp_config`.

        If you already have a fully formed configuration block
        and want to create a stage, you can call

        .. code-block:: python

            factory.registry[stage](stage_config)

        In most cases if you have a stage class object and want to
        instantiate it with its defaults you can call

        .. code-block:: python

            stage = stage_cls(stage_cls.default_config)

        .. note:: This mirrors
                  :meth:`yakonfig.factory.AutoFactory.create`, with
                  some thought that this factory class might migrate
                  to using that as a base in the future.

        :param stage: pipeline stage class, or its name in the registry
        :param dict scp_config: configuration block for the pipeline
        :param dict config: configuration block for the stage, or
          :const:`None` to get it from `scp_config`

        '''
        # Figure out what we have for a stage and its name
        if isinstance(stage, basestring):
            stage_name = stage
            stage_obj = self.registry[stage_name]
        else:
            stage_name = getattr(stage, 'config_name', stage.__name__)
            stage_obj = stage

        # Find the configuration; get a copy we can mutate
        if config is None:
            config = scp_config.get(stage_name, None)
        if config is None:
            config = getattr(stage_obj, 'default_config', {})
        config = dict(config)

        # Fill in more values
        if self.tmp_dir_suffix is None:
            config['tmp_dir_path'] = scp_config['tmp_dir_path']
        else:
            config['tmp_dir_path'] = os.path.join(scp_config['tmp_dir_path'],
                                                  self.tmp_dir_suffix)
        config['third_dir_path'] = scp_config['third_dir_path']

        return stage_obj(config)

Response: Create a pipeline stage.


Instruction:
Write a Python 3 function for
creating a list of indirect stages.

Response:
def _init_stages(self, config, name):
        '''Create a list of indirect stages.

        `name` should be the name of a config item that holds a list
        of names of stages, for instance, ``writers``.  This looks up
        the names of those stages, then creates and returns the
        corresponding list of stage objects.  For instance, if the
        config says

        .. code-block:: yaml

            incremental_transforms: [clean_html, clean_visible]

        then calling ``self._init_stages(scp_config,
        'incremental_transforms')`` will return a list of the two
        named stage instances.

        :param dict config: `streamcorpus_pipeline` configuration block
        :param str name: name of the stage name list entry
        :return: list of new stage instances

        '''
        if name not in config:
            return []
        return [self.create(stage, config) for stage in config[name]]


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _init_all_stages(self, config):
        '''Create stages that are used for the pipeline.

        :param dict config: `streamcorpus_pipeline` configuration
        :return: tuple of (reader, incremental transforms, batch
          transforms, post-batch incremental transforms, writers,
          temporary directory)

        '''
        reader = self._init_stage(config, 'reader')
        incremental_transforms = self._init_stages(
            config, 'incremental_transforms')
        batch_transforms = self._init_stages(config, 'batch_transforms')
        post_batch_incremental_transforms = self._init_stages(
            config, 'post_batch_incremental_transforms')
        writers = self._init_stages(config, 'writers')
        tmp_dir_path = os.path.join(config['tmp_dir_path'],
                                    self.tmp_dir_suffix)
        return (reader, incremental_transforms, batch_transforms,
                post_batch_incremental_transforms, writers, tmp_dir_path)

Response: Create stages that are used for the pipeline.


Instruction:
Create a Python 3 function to
process a : class : coordinate. WorkUnit.

Response:
def _process_task(self, work_unit):
        '''Process a :class:`coordinate.WorkUnit`.

        The work unit's key is taken as the input file name.  The
        data should have ``start_count`` and ``start_chunk_time``
        values, which are passed on to :meth:`run`.

        :param work_unit: work unit to process
        :paramtype work_unit: :class:`coordinate.WorkUnit`
        :return: number of stream items processed

        '''
        self.work_unit = work_unit
        i_str = work_unit.key
        start_count = work_unit.data['start_count']
        start_chunk_time = work_unit.data['start_chunk_time']
        self.run(i_str, start_count, start_chunk_time)


Instruction:
Create a Python 3 function for
running the pipeline. This runs all of the steps described in the pipeline constructor, reading from some input and writing to some output. :param str i_str: name of the input file, or other reader-specific description of where to get input :param int start_count: index of the first stream item :param int start_chunk_time: timestamp for the first stream item

Response:
def run(self, i_str, start_count=0, start_chunk_time=None):
        '''Run the pipeline.

        This runs all of the steps described in the pipeline constructor,
        reading from some input and writing to some output.

        :param str i_str: name of the input file, or other reader-specific
          description of where to get input
        :param int start_count: index of the first stream item
        :param int start_chunk_time: timestamp for the first stream item

        '''
        try:
            if not os.path.exists(self.tmp_dir_path):
                os.makedirs(self.tmp_dir_path)

            if start_chunk_time is None:
                start_chunk_time = time.time()

            ## the reader returns generators of StreamItems
            i_chunk = self.reader(i_str)

            ## t_path points to the currently in-progress temp chunk
            t_path = None

            ## loop over all docs in the chunk processing and cutting
            ## smaller chunks if needed

            len_clean_visible = 0
            sources = set()
            next_idx = 0

            ## how many have we input and actually done processing on?
            input_item_count = 0

            for si in i_chunk:
                # TODO: break out a _process_stream_item function?
                next_idx += 1

                ## yield to the gevent hub to allow other things to run
                if gevent:
                    gevent.sleep(0)

                ## skip forward until we reach start_count
                if next_idx <= start_count:
                    continue

                if next_idx % self.rate_log_interval == 0:
                    ## indexing is zero-based, so next_idx corresponds
                    ## to length of list of SIs processed so far
                    elapsed = time.time() - start_chunk_time
                    if elapsed > 0:
                        rate = float(next_idx) / elapsed
                        logger.info('%d in %.1f --> %.1f per sec on '
                                    '(pre-partial_commit) %s',
                                    next_idx - start_count, elapsed, rate,
                                    i_str)

                if not self.t_chunk:
                    ## make a temporary chunk at a temporary path
                    # (Lazy allocation after we've read an item that might get processed out to the new chunk file)
                    # TODO: make this EVEN LAZIER by not opening the t_chunk until inside _run_incremental_transforms whe the first output si is ready
                    t_path = os.path.join(self.tmp_dir_path,
                                          't_chunk-%s' % uuid.uuid4().hex)
                    self.t_chunk = streamcorpus.Chunk(path=t_path, mode='wb')
                    assert self.t_chunk.message == streamcorpus.StreamItem_v0_3_0, self.t_chunk.message

                # TODO: a set of incremental transforms is equivalent
                # to a batch transform.  Make the pipeline explicitly
                # configurable as such:
                #
                # batch_transforms: [[incr set 1], batch op, [incr set 2], ...]
                #
                # OR: for some list of transforms (mixed incremental
                # and batch) pipeline can detect and batchify as needed

                ## incremental transforms populate t_chunk
                ## let the incremental transforms destroy the si by
                ## returning None
                si = self._run_incremental_transforms(
                    si, self.incremental_transforms)

                ## insist that every chunk has only one source string
                if si:
                    sources.add(si.source)
                    if self.assert_single_source and len(sources) != 1:
                        raise InvalidStreamItem(
                            'stream item %r had source %r, not %r '
                            '(set assert_single_source: false to suppress)' %
                            (si.stream_id, si.source, sources))

                if si and si.body and si.body.clean_visible:
                    len_clean_visible += len(si.body.clean_visible)
                    ## log binned clean_visible lengths, for quick stats estimates
                    #logger.debug('len(si.body.clean_visible)=%d' % int(10 * int(math.floor(float(len(si.body.clean_visible)) / 2**10)/10)))
                    #logger.debug('len(si.body.clean_visible)=%d' % len(si.body.clean_visible))

                if ((self.output_chunk_max_count is not None and
                     len(self.t_chunk) == self.output_chunk_max_count)):
                    logger.info('reached output_chunk_max_count (%d) at: %d',
                                len(self.t_chunk), next_idx)
                    self._process_output_chunk(
                        start_count, next_idx, sources, i_str, t_path)
                    start_count = next_idx

                elif (self.output_max_clean_visible_bytes is not None and
                      len_clean_visible >=
                      self.output_chunk_max_clean_visible_bytes):
                    logger.info(
                        'reached output_chunk_max_clean_visible_bytes '
                        '(%d) at: %d',
                        self.output_chunk_max_clean_visible_bytes,
                        len_clean_visible)
                    len_clean_visible = 0
                    self._process_output_chunk(
                        start_count, next_idx, sources, i_str, t_path)
                    start_count = next_idx

                input_item_count += 1
                if (((self.input_item_limit is not None) and
                     (input_item_count > self.input_item_limit))):
                    break

            if self.t_chunk is not None:
                self._process_output_chunk(
                    start_count, next_idx, sources, i_str, t_path)

            ## return how many stream items we processed
            return next_idx

        finally:
            if self.t_chunk is not None:
                self.t_chunk.close()
            for transform in self.batch_transforms:
                transform.shutdown()
            if self.cleanup_tmp_files:
                rmtree(self.tmp_dir_path)


Instruction:
Explain what the following Python 3 code does
def _run_writers(self, start_count, next_idx, sources, i_str, t_path):
        '''Run all of the writers over some intermediate chunk.

        :param int start_count: index of the first item
        :param int next_idx: index of the next item (after the last
          item in this chunk)
        :param list sources: source strings included in this chunk
          (usually only one source)
        :param str i_str: name of input file or other input
        :param str t_path: location of intermediate chunk on disk
        :return: list of output file paths or other outputs

        '''
        # writers put the chunk somewhere, and could delete it
        name_info = dict(
            first=start_count,
            # num and md5 computed in each writers
            source=sources.pop(),
            )

        all_o_paths = []
        for writer in self.writers:
            logger.debug('running %r on %r: %r', writer, i_str, name_info)
            o_paths = writer(t_path, name_info, i_str)
            logger.debug('loaded (%d, %d) of %r into %r',
                         start_count, next_idx - 1, i_str, o_paths)
            all_o_paths += o_paths
        return all_o_paths

Response: Run all of the writers over some intermediate chunk.


Instruction:
Here you have a function in Python 3, explain what it does
def _run_incremental_transforms(self, si, transforms):
        '''
        Run transforms on stream item.
        Item may be discarded by some transform.
        Writes successful items out to current self.t_chunk
        Returns transformed item or None.
        '''
        ## operate each transform on this one StreamItem
        for transform in transforms:
            try:
                stream_id = si.stream_id
                si_new = transform(si, context=self.context)

                if si_new is None:
                    logger.warn('transform %r deleted %s abs_url=%r',
                                 transform, stream_id, si and si.abs_url)
                    return None
                si = si_new

            except TransformGivingUp:
                ## do nothing
                logger.info('transform %r giving up on %r',
                            transform, si.stream_id)

            except Exception, exc:
                logger.critical(
                    'transform %r failed on %r from i_str=%r abs_url=%r',
                    transform, si and si.stream_id, self.context.get('i_str'),
                    si and si.abs_url, exc_info=True)

        assert si is not None

        ## expect to always have a stream_time
        if not si.stream_time:
            raise InvalidStreamItem('empty stream_time: %s' % si)

        if si.stream_id is None:
            raise InvalidStreamItem('empty stream_id: %r' % si)

        ## put the StreamItem into the output
        if type(si) != streamcorpus.StreamItem_v0_3_0:
            raise InvalidStreamItem('incorrect stream item object %r' %
                                    type(si))
        self.t_chunk.add(si)
        return si

Response: Run the incremental transforms on a stream item.


Instruction:
Create a Python 3 function to
return a dict that contains the date_hour md5 num_now date_hour md5 num_now date_hour time_now date_now date_

Response:
def get_name_info(chunk_path, assert_one_date_hour=False, i_str=None,
                  chunk_type=Chunk):
    '''
    takes a chunk blob and obtains the date_hour, md5, num

    makes fields:
    i_str
    input_fname
    input_md5 - parsed from input filename if it contains '-%(md5)s-'
    md5
    num
    epoch_ticks
    target_names
    doc_ids_8
    date_hour
    rand8
    date_now
    time_now
    date_time_now
    '''
    assert i_str is not None, 'must provide i_str as keyword arg'

    name_info = dict()
    if i_str:
        name_info['i_str'] = i_str
    else:
        name_info['i_str'] = ''

    i_fname = i_str.split('/')[-1]
    i_fname = i_fname.split('.')[0]  ## strip off .sc[.xz[.gpg]]
    name_info['input_fname'] = i_fname

    input_md5s = []
    for part in i_fname.split('-'):
        if len(part) == 32 and is_hex_32.match(part):
            input_md5s.append(part)
    name_info['input_md5'] = '-'.join(input_md5s)

    # TODO: return a dict-like object that does the expensive
    # calculation lazily, the name format might not even need that
    # value.
    ch = chunk_type(path=chunk_path, mode='rb')
    date_hours = set()
    target_names = set()
    doc_ids = set()
    epoch_ticks = None
    count = 0
    try:
        for si in ch:
            if chunk_type is Chunk:
                if epoch_ticks is None:
                    epoch_ticks = si.stream_time.epoch_ticks
                date_hours.add( si.stream_time.zulu_timestamp[:13] )
                doc_ids.add( si.doc_id )
                for annotator_id, ratings in si.ratings.items():
                    for rating in ratings:
                        target_name = rating.target.target_id.split('/')[-1]
                        target_names.add( target_name )
            count += 1
    except Exception, exc:
        logger.critical('failed to iter over chunk', exc_info=True)

    ## create the md5 property, so we can use it in the filename
    if hasattr(ch, 'md5_hexdigest'):
        name_info['md5'] = ch.md5_hexdigest
    else:
        try:
            data = open(chunk_path).read()
            name_info['md5'] = hashlib.md5(data).hexdigest()
        except Exception, exc:
            logger.critical('failed to compute md5', exc_info=True)
            name_info['md5'] = 'broken'
    name_info['num'] = count
    name_info['epoch_ticks'] = epoch_ticks

    name_info['target_names'] = '-'.join( target_names )
    name_info['doc_ids_8'] = '-'.join( [di[:8] for di in doc_ids] )

    if chunk_type is Chunk:
        if assert_one_date_hour:
            assert len(date_hours) == 1, \
                'got a chunk with other than one data_hour! ' + \
                repr(date_hours)

        if len(date_hours) > 0:
            date_hour = list(date_hours)[0]
            date_hour = date_hour.replace('T', '-')
        else:
            assert count == 0, (date_hours, count)
            date_hour = None
        name_info['date_hour'] = date_hour
    else:
        name_info['date_hour'] = 'NO-DATE-HOUR-FOR-FC'

    # TODO: in future lazy evaluation world, rand8 should return a
    # different value every time it is accessed so that a format could
    # be 'foo-{rand8}{rand8}'
    name_info['rand8'] = '%08x' % (random.randint(0, 0x7fffffff),)

    name_info['date_now'] = datetime.datetime.utcnow().strftime('%Y-%m-%d')
    name_info['time_now'] = datetime.datetime.utcnow().strftime('%H-%M-%S')
    name_info['date_time_now'] = datetime.datetime.utcnow().strftime('%Y-%m-%d-%H-%M-%S')

    return name_info


Instruction:
Can you implement a function in Python 3 that
replaces the top - level pipeline configurable object.

Response:
def replace_config(config, name):
    '''Replace the top-level pipeline configurable object.

    This investigates a number of sources, including
    `external_stages_path` and `external_stages_modules` configuration
    and `streamcorpus_pipeline.stages` entry points, and uses these to
    find the actual :data:`sub_modules` for
    :mod:`streamcorpus_pipeline`.

    '''
    global static_stages
    if static_stages is None:
        static_stages = PipelineStages()
        stages = static_stages
        if 'external_stages_path' in config:
            path = config['external_stages_path']
            if not os.path.isabs(path) and config.get('root_path'):
                path = os.path.join(config['root_path'], path)
            try:
                stages.load_external_stages(config['external_stages_path'])
            except IOError:
                return streamcorpus_pipeline  # let check_config re-raise this
        if 'external_stages_modules' in config:
            for mod in config['external_stages_modules']:
                try:
                    stages.load_module_stages(mod)
                except ImportError:
                    return streamcorpus_pipeline  # let check_config re-raise this
    else:
        stages = static_stages

    new_sub_modules = set(stage
                          for stage in stages.itervalues()
                          if hasattr(stage, 'config_name'))
    return NewSubModules(streamcorpus_pipeline, new_sub_modules)


Instruction:
Can you generate the documentation for the following Python 3 function
def make_app():
    env = Environment()
    # STDIN is ignored because HTTPony runs a server that doesn't care.
    # Additionally, it is needed or else pytest blows up.
    args = parser.parse_args(args=['/', '--ignore-stdin'], env=env)
    args.output_options = 'HB'  # Output only requests.
    server = 'HTTPony/{0}'.format(__version__)

    def application(environ, start_response):
        # The WSGI server puts content length and type in the environment
        # even when not provided with the request. Drop them if they are empty.
        if environ.get('CONTENT_LENGTH') == '':
            del environ['CONTENT_LENGTH']
        if environ.get('CONTENT_TYPE') == '':
            del environ['CONTENT_TYPE']

        wrequest = WerkzeugRequest(environ)
        data = wrequest.get_data()
        request = Request(
            method=wrequest.method,
            url=wrequest.url,
            headers=wrequest.headers,
            data=data,
        )
        prepared = request.prepare()

        stream = streams.build_output_stream(
            args, env, prepared, response=None,
            output_options=args.output_options)
        streams.write_stream(stream, env.stdout, env.stdout_isatty)

        # When there is data in the request, give the next one breathing room.
        if data:
            print("\n", file=env.stdout)

        # Make dreams come true.
        response = Response(headers={'Server': server})
        return response(environ, start_response)

    return application

Response: Make a WSGI application that has all the HTTPie pieces baked in.