Instruction:
Explain what the following Python 3 code does
def plot_aligned(pca, sparse=True, **kwargs):
    colormap = kwargs.pop('colormap',None)

    A = pca.rotated()
    # Flip the result if upside down
    if pca.normal[2] < 0:
        A[:,-1] *= -1
    minmax = [(A[:,i].min(),A[:,i].max()) for i in range(3)]
    lengths = [j-i for i,j in minmax]

    if sparse:
        i = 1
        l = len(A)
        if l > 10000:
            i = N.ceil(l/10000)
            A = A[::int(i)]

    log.info("Plotting with {} points".format(len(A)))

    w = 8
    ratio = (lengths[2]*2+lengths[1])/lengths[0]*1.5
    h = w*ratio
    if h < 3: h = 3

    r = (lengths[1]+5)/(lengths[2]+5)
    if r > 5:
        r = 5

    fig = Figure(figsize=(w,h))
    fig.canvas = FigureCanvas(fig)

    def setup_axes():
        gs = GridSpec(3,1, height_ratios=[r,1,1])
        kwargs = dict()
        axes = []
        for g in gs:
            ax = fig.add_subplot(g,**kwargs)
            kwargs['sharex'] = ax
            yield ax

    axes = list(setup_axes())

    fig.subplots_adjust(hspace=0, wspace=0.1)

    #lengths = attitude.pca.singular_values[::-1]

    titles = (
        "Plan view",
        "Long cross-section",
        "Short cross-section")

    ylabels = (
        "Meters",
        "Residuals (m)",
        "Residuals (m)")

    colors = ['cornflowerblue','red']

    hyp = sampling_axes(pca)
    vertical = vector(0,0,1)

    for title,ax,(a,b),ylabel in zip(titles,axes,
            [(0,1),(0,2),(1,2)],ylabels):

        kw = dict(linewidth=2, alpha=0.5)
        bounds = minmax[a]
        if b != 2:
            ax.plot(bounds,(0,0), c=colors[a], **kw)
            ax.plot((0,0),minmax[b], c=colors[b], **kw)
        else:
            ax.plot(bounds,(-10,-10), c=colors[a], **kw)
            v0 = N.zeros(3)
            v0[a] = 1
            axes = N.array([v0,vertical])

            ax_ = (axes@N.diag(hyp)@axes.T).T
            ax_ = N.sqrt(ax_)
            l1 = N.linalg.norm(ax_[:-1])
            l2 = N.linalg.norm(ax_[-1])
            ang_error = 2*N.degrees(N.arctan2(l2,l1))

            title += ": {:.0f} m long, angular error (95% CI): {:.2f}º".format(lengths[a], ang_error)

            bounds = minmax[0]
            x_ = N.linspace(bounds[0]*1.2,bounds[1]*1.2,100)

            err = HyperbolicErrors(hyp,x_,axes=axes)
            err.plot(ax, fc='#cccccc', alpha=0.3)

        x,y = A[:,a], A[:,b]
        kw = dict(alpha=0.5, zorder=5)

        if colormap is None:
            ax.plot(x,y,c="#555555", linestyle='None', marker='.',**kw)
        else:
            ax.scatter(x,y,c=A[:,-1], cmap=colormap, **kw)

        #ax.set_aspect("equal")

        ax.text(0.01,.99,title,
            verticalalignment='top',
            transform=ax.transAxes)
        #ax.autoscale(tight=True)
        ax.yaxis.set_ticks([])
        ax.xaxis.set_ticks_position('bottom')
        if a != 1:
            pass
            #ax.xaxis.set_ticks([])
            #ax.spines['bottom'].set_color('none')
        for spine in ax.spines.values():
            spine.set_visible(False)

    ax.text(0.99,0.99,"Max residual: {:.1f} m".format(lengths[2]),
        verticalalignment='bottom',
        ha='right',
        transform=ax.transAxes)

    ax.set_xlabel("Meters")
    return fig

Response: Plots the residuals of a principal component in the same direction as the given principal component.


Instruction:
Write a Python 3 script to
map each osm POI and geotweets based on spatial lookup of metro area

Response:
def mapper_metro(self, _, data):
        """ map each osm POI and geotweets based on spatial lookup of metro area """
        # OSM POI record
        if 'tags' in data:
            type_tag = 1
            lonlat = data['coordinates']
            payload = data['tags']
        # Tweet with coordinates from Streaming API
        elif 'user_id' in data:
            type_tag = 2
            # only allow tweets from the listed domains to try and filter out
            # noise such as HR tweets, Weather reports and news updates
            accept = [
                "twitter\.com",
                "foursquare\.com",
                "instagram\.com",
                "untappd\.com"
            ]
            expr = "|".join(accept)
            if not re.findall(expr, data['source']):
                return
            lonlat = data['lonlat']
            payload = None
        # spatial lookup using Rtree with cached results
        metro = self.lookup.get(lonlat, METRO_DISTANCE)
        if not metro:
            return
        yield metro, (type_tag, lonlat, payload)


Instruction:
Can you implement a function in Python 3 that
outputs tags of POI locations nearby tweet locations Values will be sorted coming into reducer. First element in each value tuple will be either 1 (osm POI) or 2 (geotweet). Build a spatial index with POI records. For each tweet lookup nearby POI, and emit tag values for predefined tags.

Response:
def reducer_metro(self, metro, values):
        """
        Output tags of POI locations nearby tweet locations

        Values will be sorted coming into reducer.
        First element in each value tuple will be either 1 (osm POI) or 2 (geotweet).
        Build a spatial index with POI records.
        For each tweet lookup nearby POI, and emit tag values for predefined tags.

        """
        lookup = CachedLookup(precision=POI_GEOHASH_PRECISION)
        for i, value in enumerate(values):
            type_tag, lonlat, data = value
            if type_tag == 1:
                # OSM POI node, construct geojson and add to Rtree index
                lookup.insert(i, dict(
                    geometry=dict(type='Point', coordinates=project(lonlat)),
                    properties=dict(tags=data)
                ))
            else:
                # geotweet, lookup nearest POI from index
                if not lookup.data_store:
                    return
                poi_names = []
                kwargs = dict(buffer_size=POI_DISTANCE, multiple=True)
                # lookup nearby POI from Rtree index (caching results)
                # for any tags we care about emit the tags value and 1
                for poi in lookup.get(lonlat, **kwargs):
                    has_tag = [ tag in poi['tags'] for tag in POI_TAGS ]
                    if any(has_tag) and 'name' in poi['tags']:
                        poi_names.append(poi['tags']['name'])
                for poi in set(poi_names):
                    yield (metro, poi), 1


Instruction:
Can you generate a brief explanation for the following Python 3 code
def reducer_count(self, key, values):
        total = sum(values)
        metro, poi = key
        # group data by metro areas for final output
        yield metro, (total, poi)

Response: count occurences for each metro record


Instruction:
Here you have a function in Python 3, explain what it does
def reducer_output(self, metro, values):
        records = []
        # build up list of data for each metro area and submit as one network
        # call instead of individually
        for value in values:
            total, poi = value
            records.append(dict(
                metro_area=metro,
                poi=poi,
                count=total
            ))
            output = "{0}\t{1}\t{2}"
            output = output.format(metro.encode('utf-8'), total, poi.encode('utf-8'))
            yield None, output
        if self.mongo:
            self.mongo.insert_many(records)

Response: store each record in MongoDB and output tab delimited lines


Instruction:
Create a Python 3 function to
raise an exception if the SQLCompiler is not bound to an unknown dialect.

Response:
def fail_unknown_dialect(compiler: "SQLCompiler", task: str) -> None:
    """
    Raise :exc:`NotImplementedError` in relation to a dialect for which a
    function hasn't been implemented (with a helpful error message).
    """
    raise NotImplementedError(
        "Don't know how to {task} on dialect {dialect!r}. "
        "(Check also: if you printed the SQL before it was bound to an "
        "engine, you will be trying to use a dialect like StrSQLCompiler, "
        "which could be a reason for failure.)".format(
            task=task,
            dialect=compiler.dialect
        )
    )


Instruction:
Can you generate a brief explanation for the following Python 3 code
def fetch_processed_single_clause(element: "ClauseElement",
                                  compiler: "SQLCompiler") -> str:
    """
    Takes a clause element that must have a single clause, and converts it
    to raw SQL text.
    """
    if len(element.clauses) != 1:
        raise TypeError("Only one argument supported; {} were passed".format(
            len(element.clauses)))
    clauselist = element.clauses  # type: ClauseList
    first = clauselist.get_children()[0]
    return compiler.process(first)

Response: Takes a clause element that must have a single clause and converts it to raw SQL text.


Instruction:
Implement a Python 3 function for
converting a parameter to an integer or raises a forms. ValidationError.

Response:
def clean_int(x) -> int:
    """
    Returns its parameter as an integer, or raises
    ``django.forms.ValidationError``.
    """
    try:
        return int(x)
    except ValueError:
        raise forms.ValidationError(
            "Cannot convert to integer: {}".format(repr(x)))


Instruction:
Can you generate the documentation for the following Python 3 function
def clean_nhs_number(x) -> int:
    try:
        x = int(x)
        if not is_valid_nhs_number(x):
            raise ValueError
        return x
    except ValueError:
        raise forms.ValidationError(
            "Not a valid NHS number: {}".format(repr(x)))

Response: Returns its parameter as a valid integer NHS number or raises a forms. ValidationError.


Instruction:
Can you tell what is the following Python 3 function doing
def coltype_as_typeengine(coltype: Union[VisitableType,
                                         TypeEngine]) -> TypeEngine:
    """
    Instances of SQLAlchemy column types are subclasses of ``TypeEngine``.
    It's possible to specify column types either as such instances, or as the
    class type. This function ensures that such classes are converted to
    instances.

    To explain: you can specify columns like

    .. code-block:: python

        a = Column("a", Integer)
        b = Column("b", Integer())
        c = Column("c", String(length=50))

        isinstance(Integer, TypeEngine)  # False
        isinstance(Integer(), TypeEngine)  # True
        isinstance(String(length=50), TypeEngine)  # True

        type(Integer)  # <class 'sqlalchemy.sql.visitors.VisitableType'>
        type(Integer())  # <class 'sqlalchemy.sql.sqltypes.Integer'>
        type(String)  # <class 'sqlalchemy.sql.visitors.VisitableType'>
        type(String(length=50))  # <class 'sqlalchemy.sql.sqltypes.String'>

    This function coerces things to a :class:`TypeEngine`.
    """
    if isinstance(coltype, TypeEngine):
        return coltype
    return coltype()

Response: Returns the given column type as a typeengine.


Instruction:
Given the following Python 3 function, write the documentation
def walk_orm_tree(obj,
                  debug: bool = False,
                  seen: Set = None,
                  skip_relationships_always: List[str] = None,
                  skip_relationships_by_tablename: Dict[str, List[str]] = None,
                  skip_all_relationships_for_tablenames: List[str] = None,
                  skip_all_objects_for_tablenames: List[str] = None) \
        -> Generator[object, None, None]:
    """
    Starting with a SQLAlchemy ORM object, this function walks a
    relationship tree, yielding each of the objects once.

    To skip attributes by name, put the attribute name(s) in
    ``skip_attrs_always``. To skip by table name, pass
    ``skip_attrs_by_tablename`` as e.g.

    .. code-block:: python

        {'sometable': ['attr1_to_skip', 'attr2_to_skip']}

    Args:
        obj: the SQLAlchemy ORM object to walk

        debug: be verbose

        seen: usually ``None``, but can be a set of objects marked as "already
            seen"; if an object is in this set, it is skipped

        skip_relationships_always: relationships are skipped if the
            relationship has a name in this (optional) list

        skip_relationships_by_tablename: optional dictionary mapping table
            names (keys) to relationship attribute names (values); if the
            "related table"/"relationship attribute" pair are in this
            dictionary, the relationship is skipped

        skip_all_relationships_for_tablenames: relationships are skipped if the
            the related table has a name in this (optional) list

        skip_all_objects_for_tablenames: if the object belongs to a table whose
            name is in this (optional) list, the object is skipped

    Yields:
        SQLAlchemy ORM objects (including the starting object)

    """
    # http://docs.sqlalchemy.org/en/latest/faq/sessions.html#faq-walk-objects
    skip_relationships_always = skip_relationships_always or []  # type: List[str]  # noqa
    skip_relationships_by_tablename = skip_relationships_by_tablename or {}  # type: Dict[str, List[str]]  # noqa
    skip_all_relationships_for_tablenames = skip_all_relationships_for_tablenames or []  # type: List[str]  # noqa
    skip_all_objects_for_tablenames = skip_all_objects_for_tablenames or []  # type: List[str]  # noqa
    stack = [obj]
    if seen is None:
        seen = set()
    while stack:
        obj = stack.pop(0)
        if obj in seen:
            continue
        tablename = obj.__tablename__
        if tablename in skip_all_objects_for_tablenames:
            continue
        seen.add(obj)
        if debug:
            log.debug("walk: yielding {!r}", obj)
        yield obj
        insp = inspect(obj)  # type: InstanceState
        for relationship in insp.mapper.relationships:  # type: RelationshipProperty  # noqa
            attrname = relationship.key
            # Skip?
            if attrname in skip_relationships_always:
                continue
            if tablename in skip_all_relationships_for_tablenames:
                continue
            if (tablename in skip_relationships_by_tablename and
                    attrname in skip_relationships_by_tablename[tablename]):
                continue
            # Process relationship
            if debug:
                log.debug("walk: following relationship {}", relationship)
            related = getattr(obj, attrname)
            if debug and related:
                log.debug("walk: queueing {!r}", related)
            if relationship.uselist:
                stack.extend(related)
            elif related is not None:
                stack.append(related)

Response: This function walks the SQLAlchemy ORM object tree and yields each of the objects in the order they appear in the SQLAlchemy ORM object tree.


Instruction:
Can you create a Python 3 function that
gives an SQLAlchemy object creates a new object and copies all properties of the object into a new object.

Response:
def copy_sqla_object(obj: object,
                     omit_fk: bool = True,
                     omit_pk: bool = True,
                     omit_attrs: List[str] = None,
                     debug: bool = False) -> object:
    """
    Given an SQLAlchemy object, creates a new object (FOR WHICH THE OBJECT
    MUST SUPPORT CREATION USING ``__init__()`` WITH NO PARAMETERS), and copies
    across all attributes, omitting PKs (by default), FKs (by default), and
    relationship attributes (always omitted).

    Args:
        obj: the object to copy
        omit_fk: omit foreign keys (FKs)?
        omit_pk: omit primary keys (PKs)?
        omit_attrs: attributes (by name) not to copy
        debug: be verbose

    Returns:
        a new copy of the object
    """
    omit_attrs = omit_attrs or []  # type: List[str]
    cls = type(obj)
    mapper = class_mapper(cls)
    newobj = cls()  # not: cls.__new__(cls)
    rel_keys = set([c.key for c in mapper.relationships])
    prohibited = rel_keys
    if omit_pk:
        pk_keys = set([c.key for c in mapper.primary_key])
        prohibited |= pk_keys
    if omit_fk:
        fk_keys = set([c.key for c in mapper.columns if c.foreign_keys])
        prohibited |= fk_keys
    prohibited |= set(omit_attrs)
    if debug:
        log.debug("copy_sqla_object: skipping: {}", prohibited)
    for k in [p.key for p in mapper.iterate_properties
              if p.key not in prohibited]:
        try:
            value = getattr(obj, k)
            if debug:
                log.debug("copy_sqla_object: processing attribute {} = {}",
                          k, value)
            setattr(newobj, k, value)
        except AttributeError:
            if debug:
                log.debug("copy_sqla_object: failed attribute {}", k)
            pass
    return newobj


Instruction:
Can you generate a brief explanation for the following Python 3 code
def deepcopy_sqla_objects(
        startobjs: List[object],
        session: Session,
        flush: bool = True,
        debug: bool = False,
        debug_walk: bool = True,
        debug_rewrite_rel: bool = False,
        objmap: Dict[object, object] = None) -> None:
    """
    Makes a copy of the specified SQLAlchemy ORM objects, inserting them into a
    new session.

    This function operates in several passes:

    1. Walk the ORM tree through all objects and their relationships, copying
       every object thus found (via :func:`copy_sqla_object`, without their
       relationships), and building a map from each source-session object to
       its equivalent destination-session object.

    2. Work through all the destination objects, rewriting their relationships
       (via :func:`rewrite_relationships`) so they relate to each other (rather
       than their source-session brethren).

    3. Insert all the destination-session objects into the destination session.

    For this to succeed, every object must take an ``__init__`` call with no
    arguments (see :func:`copy_sqla_object`). (We can't specify the required
    ``args``/``kwargs``, since we are copying a tree of arbitrary objects.)

    Args:
        startobjs: SQLAlchemy ORM objects to copy
        session: destination SQLAlchemy :class:`Session` into which to insert
            the copies
        flush: flush the session when we've finished?
        debug: be verbose?
        debug_walk: be extra verbose when walking the ORM tree?
        debug_rewrite_rel: be extra verbose when rewriting relationships?
        objmap: starting object map from source-session to destination-session
            objects (see :func:`rewrite_relationships` for more detail);
            usually ``None`` to begin with.
    """
    if objmap is None:
        objmap = {}  # keys = old objects, values = new objects
    if debug:
        log.debug("deepcopy_sqla_objects: pass 1: create new objects")

    # Pass 1: iterate through all objects. (Can't guarantee to get
    # relationships correct until we've done this, since we don't know whether
    # or where the "root" of the PK tree is.)
    seen = set()
    for startobj in startobjs:
        for oldobj in walk_orm_tree(startobj, seen=seen, debug=debug_walk):
            if debug:
                log.debug("deepcopy_sqla_objects: copying {}", oldobj)
            newobj = copy_sqla_object(oldobj, omit_pk=True, omit_fk=True)
            # Don't insert the new object into the session here; it may trigger
            # an autoflush as the relationships are queried, and the new
            # objects are not ready for insertion yet (as their relationships
            # aren't set).
            # Note also the session.no_autoflush option:
            # "sqlalchemy.exc.OperationalError: (raised as a result of Query-
            # invoked autoflush; consider using a session.no_autoflush block if
            # this flush is occurring prematurely)..."
            objmap[oldobj] = newobj

    # Pass 2: set all relationship properties.
    if debug:
        log.debug("deepcopy_sqla_objects: pass 2: set relationships")
    for oldobj, newobj in objmap.items():
        if debug:
            log.debug("deepcopy_sqla_objects: newobj: {}", newobj)
        rewrite_relationships(oldobj, newobj, objmap, debug=debug_rewrite_rel)

    # Now we can do session insert.
    if debug:
        log.debug("deepcopy_sqla_objects: pass 3: insert into session")
    for newobj in objmap.values():
        session.add(newobj)

    # Done
    if debug:
        log.debug("deepcopy_sqla_objects: done")
    if flush:
        session.flush()

Response: This function copies SQLAlchemy ORM objects into a new SQLAlchemy ORM object tree.


Instruction:
Given the following Python 3 function, write the documentation
def deepcopy_sqla_object(startobj: object,
                         session: Session,
                         flush: bool = True,
                         debug: bool = False,
                         debug_walk: bool = False,
                         debug_rewrite_rel: bool = False,
                         objmap: Dict[object, object] = None) -> object:
    """
    Makes a copy of the object, inserting it into ``session``.

    Uses :func:`deepcopy_sqla_objects` (q.v.).

    A problem is the creation of duplicate dependency objects if you call it
    repeatedly.

    Optionally, if you pass the objmap in (which maps old to new objects), you
    can call this function repeatedly to clone a related set of objects...
    ... no, that doesn't really work, as it doesn't visit parents before
    children. The :func:`cardinal_pythonlib.sqlalchemy.merge_db.merge_db`
    function does that properly.

    Args:
        startobj: SQLAlchemy ORM object to deep-copy
        session: see :func:`deepcopy_sqla_objects`
        flush: see :func:`deepcopy_sqla_objects`
        debug: see :func:`deepcopy_sqla_objects`
        debug_walk: see :func:`deepcopy_sqla_objects`
        debug_rewrite_rel: see :func:`deepcopy_sqla_objects`
        objmap: see :func:`deepcopy_sqla_objects`

    Returns:
        the copied object matching ``startobj``

    """
    if objmap is None:
        objmap = {}  # keys = old objects, values = new objects
    deepcopy_sqla_objects(
        startobjs=[startobj],
        session=session,
        flush=flush,
        debug=debug,
        debug_walk=debug_walk,
        debug_rewrite_rel=debug_rewrite_rel,
        objmap=objmap
    )
    return objmap[startobj]

Response: This function copies an SQLAlchemy ORM object into a new SQLAlchemy object.


Instruction:
Explain what the following Python 3 code does
def gen_columns(obj) -> Generator[Tuple[str, Column], None, None]:
    mapper = obj.__mapper__  # type: Mapper
    assert mapper, "gen_columns called on {!r} which is not an " \
                   "SQLAlchemy ORM object".format(obj)
    colmap = mapper.columns  # type: OrderedProperties
    if not colmap:
        return
    for attrname, column in colmap.items():
        # NB: column.name is the SQL column name, not the attribute name
        yield attrname, column

Response: Asks a SQLAlchemy ORM object : what are your SQLAlchemy columns?


Instruction:
Can you tell what is the following Python 3 function doing
def get_pk_attrnames(obj) -> List[str]:
    return [attrname
            for attrname, column in gen_columns(obj)
            if column.primary_key]

Response: Asks an SQLAlchemy ORM object to be what are your primary key ( s?


Instruction:
Write a Python 3 function for
generating tuples of names and column objects for the given UNINSTRUMENTED class.

Response:
def gen_columns_for_uninstrumented_class(cls: Type) \
        -> Generator[Tuple[str, Column], None, None]:
    """
    Generate ``(attr_name, Column)`` tuples from an UNINSTRUMENTED class, i.e.
    one that does not inherit from ``declarative_base()``. Use this for mixins
    of that kind.

    SUBOPTIMAL. May produce warnings like:

    .. code-block:: none

        SAWarning: Unmanaged access of declarative attribute id from non-mapped class GenericTabletRecordMixin

    Try to use :func:`gen_columns` instead.
    """  # noqa
    for attrname in dir(cls):
        potential_column = getattr(cls, attrname)
        if isinstance(potential_column, Column):
            yield attrname, potential_column


Instruction:
Can you generate a brief explanation for the following Python 3 code
def attrname_to_colname_dict(cls) -> Dict[str, str]:
    attr_col = {}  # type: Dict[str, str]
    for attrname, column in gen_columns(cls):
        attr_col[attrname] = column.name
    return attr_col

Response: Asks an SQLAlchemy class how its attribute names correspond to database column names.


Instruction:
Explain what the following Python 3 code does
def gen_relationships(obj) -> Generator[Tuple[str, RelationshipProperty, Type],
                                        None, None]:
    """
    Yields tuples of ``(attrname, RelationshipProperty, related_class)``
    for all relationships of an ORM object.
    The object 'obj' can be EITHER an instance OR a class.
    """
    insp = inspect(obj)  # type: InstanceState
    # insp.mapper.relationships is of type
    # sqlalchemy.utils._collections.ImmutableProperties, which is basically
    # a sort of AttrDict.
    for attrname, rel_prop in insp.mapper.relationships.items():  # type: Tuple[str, RelationshipProperty]  # noqa
        # noinspection PyUnresolvedReferences
        related_class = rel_prop.mapper.class_
        # log.critical("gen_relationships: attrname={!r}, "
        #              "rel_prop={!r}, related_class={!r}, rel_prop.info={!r}",
        #              attrname, rel_prop, related_class, rel_prop.info)
        yield attrname, rel_prop, related_class

Response: Yields tuples of attrname rel_prop related_class for all relationships of an ORM object.


Instruction:
How would you implement a function in Python 3 that
gets a list of columns from an SQLAlchemy ORM class.

Response:
def get_orm_columns(cls: Type) -> List[Column]:
    """
    Gets :class:`Column` objects from an SQLAlchemy ORM class.
    Does not provide their attribute names.
    """
    mapper = inspect(cls)  # type: Mapper
    # ... returns InstanceState if called with an ORM object
    #     http://docs.sqlalchemy.org/en/latest/orm/session_state_management.html#session-object-states  # noqa
    # ... returns Mapper if called with an ORM class
    #     http://docs.sqlalchemy.org/en/latest/orm/mapping_api.html#sqlalchemy.orm.mapper.Mapper  # noqa
    colmap = mapper.columns  # type: OrderedProperties
    return colmap.values()


Instruction:
Write a Python 3 script to
get the column names from an SQLAlchemy ORM class.

Response:
def get_orm_column_names(cls: Type, sort: bool = False) -> List[str]:
    """
    Gets column names (that is, database column names) from an SQLAlchemy
    ORM class.
    """
    colnames = [col.name for col in get_orm_columns(cls)]
    return sorted(colnames) if sort else colnames


Instruction:
Can you write a function in Python 3 where it
returns all database table names found in an SQLAlchemy MetaData object.

Response:
def get_table_names_from_metadata(metadata: MetaData) -> List[str]:
    """
    Returns all database table names found in an SQLAlchemy :class:`MetaData`
    object.
    """
    return [table.name for table in metadata.tables.values()]


Instruction:
Implement a function in Python 3 to
yield all the ORM base classes except those that are abstract.

Response:
def gen_orm_classes_from_base(base: Type) -> Generator[Type, None, None]:
    """
    From an SQLAlchemy ORM base class, yield all the subclasses (except those
    that are abstract).

    If you begin with the proper :class`Base` class, then this should give all
    ORM classes in use.
    """
    for cls in gen_all_subclasses(base):
        if _get_immediate_cls_attr(cls, '__abstract__', strict=True):
            # This is SQLAlchemy's own way of detecting abstract classes; see
            # sqlalchemy.ext.declarative.base
            continue  # NOT an ORM class
        yield cls


Instruction:
Can you generate a brief explanation for the following Python 3 code
def get_orm_classes_by_table_name_from_base(base: Type) -> Dict[str, Type]:
    # noinspection PyUnresolvedReferences
    return {cls.__tablename__: cls for cls in gen_orm_classes_from_base(base)}

Response: Given an SQLAlchemy ORM base class returns a dictionary whose keys are SQLAlchemy ORM table names and whose values are ORM classes.


Instruction:
Implement a Python 3 function for
returning what looks like a plain object with the values of the SQLAlchemy ORM object.

Response:
def get_attrdict(self) -> OrderedNamespace:
        """
        Returns what looks like a plain object with the values of the
        SQLAlchemy ORM object.
        """
        # noinspection PyUnresolvedReferences
        columns = self.__table__.columns.keys()
        values = (getattr(self, x) for x in columns)
        zipped = zip(columns, values)
        return OrderedNamespace(zipped)


Instruction:
Given the following Python 3 function, write the documentation
def from_attrdict(cls, attrdict: OrderedNamespace) -> object:
        dictionary = attrdict.__dict__
        # noinspection PyArgumentList
        return cls(**dictionary)

Response: Builds an ORM object from an attribute dictionary.


Instruction:
Create a Python 3 function for
returning True if the derived class implements a method of the given name.

Response:
def derived_class_implements_method(derived: Type[T1],
                                    base: Type[T2],
                                    method_name: str) -> bool:
    """
    Does a derived class implement a method (and not just inherit a base
    class's version)?

    Args:
        derived: a derived class
        base: a base class
        method_name: the name of a method

    Returns:
        whether the derived class method is (a) present, and (b) different to
        the base class's version of the same method

    Note: if C derives from B derives from A, then a check on C versus A will
    succeed if C implements the method, or if C inherits it from B but B has
    re-implemented it compared to A.

    """
    derived_method = getattr(derived, method_name, None)
    if derived_method is None:
        return False
    base_method = getattr(base, method_name, None)
    # if six.PY2:
    #     return derived_method.__func__ != base_method.__func__
    # else:
    #     return derived_method is not base_method
    return derived_method is not base_method


Instruction:
Implement a Python 3 function for
generating all subclasses of a class.

Response:
def gen_all_subclasses(cls: Type) -> Generator[Type, None, None]:
    """
    Generates all subclasses of a class.

    Args:
        cls: a class

    Yields:
        all subclasses

    """

    for s1 in cls.__subclasses__():
        yield s1
        for s2 in gen_all_subclasses(s1):
            yield s2


Instruction:
Given the following Python 3 function, write the documentation
def augment_tensor(matrix, ndim=None):
    s = matrix.shape
    if ndim is None:
        ndim = s[0]+1
    arr = N.identity(ndim)
    arr[:s[0],:s[1]] = matrix
    return arr

Response: Augment a tensor with an identity matrix.


Instruction:
Can you tell what is the following Python 3 function doing
def angle(v1,v2, cos=False):
    n = (norm(v1)*norm(v2))
    _ = dot(v1,v2)/n
    return _ if cos else N.arccos(_)

Response: Find the angle between two vectors.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def perpendicular_vector(n):
    dim = len(n)
    if dim == 2:
        return n[::-1]
    # More complex in 3d
    for ix in range(dim):
        _ = N.zeros(dim)
        # Try to keep axes near the global projection
        # by finding vectors perpendicular to higher-
        # index axes first. This may or may not be worth
        # doing.
        _[dim-ix-1] = 1
        v1 = N.cross(n,_)
        if N.linalg.norm(v1) != 0:
            return v1
    raise ValueError("Cannot find perpendicular vector")

Response: Get a random vector perpendicular to the given vector


Instruction:
Can you implement a function in Python 3 that
traps Ctrl - C and Ctrl - B signals from doing Ctrl - C.

Response:
def trap_ctrl_c_ctrl_break() -> None:
    """
    Prevent ``CTRL-C``, ``CTRL-BREAK``, and similar signals from doing
    anything.

    See

    - https://docs.python.org/3/library/signal.html#signal.SIG_IGN
    - https://msdn.microsoft.com/en-us/library/xdkz3x12.aspx
    - https://msdn.microsoft.com/en-us/library/windows/desktop/ms682541(v=vs.85).aspx

    Under Windows, the only options are:

      =========== ======================= =====================================
      Signal      Meaning                 Comment
      =========== ======================= =====================================
      SIGABRT     abnormal termination
      SIGFPE      floating-point error
      SIGILL      illegal instruction
      SIGINT      CTRL+C signal           -- trapped here
      SIGSEGV     illegal storage access
      SIGTERM     termination request     -- trapped here
      SIGBREAK    CTRL+BREAK              -- trapped here under Windows
      =========== ======================= =====================================

    In Linux, you also find:

      =========== =============================
      Signal      Meaning
      =========== =============================
      SIGBUS      bus error / unaligned access
      =========== =============================

    To ignore, can do:

    .. code-block:: python

      signal.signal(signal.SIGINT, signal.SIG_IGN)  # SIG_IGN = "ignore me"

    or pass a specified handler, as in the code here.
    """  # noqa

    signal.signal(signal.SIGINT, ctrl_c_trapper)
    signal.signal(signal.SIGTERM, sigterm_trapper)
    if platform.system() == 'Windows':
        # SIGBREAK isn't in the Linux signal module
        # noinspection PyUnresolvedReferences
        signal.signal(signal.SIGBREAK, ctrl_break_trapper)


Instruction:
Can you generate the documentation for the following Python 3 function
def is_openxml_good(filename: str) -> bool:
    try:
        log.debug("Trying: {}", filename)
        with ZipFile(filename, 'r') as zip_ref:
            namelist = zip_ref.namelist()  # type: List[str]
            # log.critical("\n{}", pformat(namelist))
            # -----------------------------------------------------------------
            # Contains key files?
            # -----------------------------------------------------------------
            for mandatory_filename in MANDATORY_FILENAMES:
                if mandatory_filename not in namelist:
                    log.debug("Bad [missing {!r}]: {}",
                              mandatory_filename, filename)
                    return False

            infolist = zip_ref.infolist()  # type: List[ZipInfo]
            contains_docx = False
            contains_pptx = False
            contains_xlsx = False
            for info in infolist:
                # -------------------------------------------------------------
                # Sensible date check?
                # ... NO: lots of perfectly good files have this date/time.
                # -------------------------------------------------------------
                # if info.date_time == NULL_DATE_TIME:
                #     log.debug("{!r}: {!r}", info.filename, info.date_time)

                # -------------------------------------------------------------
                # Only one kind of contents?
                # ... YES, I think so. This has 100% reliability on my
                # stash of 34 PPTX, 223 DOCX, 85 XLSX, and labelled none as bad
                # from an HFC collection of 1866 such files. There are lots of
                # files emerging from Scalpel (plus my find_recovered_openxml
                # zip-fixing tool) that fail this test, though.
                # -------------------------------------------------------------
                if (not contains_docx and
                        DOCX_CONTENTS_REGEX.search(info.filename)):
                    contains_docx = True
                if (not contains_pptx and
                        PPTX_CONTENTS_REGEX.search(info.filename)):
                    contains_pptx = True
                if (not contains_xlsx and
                        XLSX_CONTENTS_REGEX.search(info.filename)):
                    contains_xlsx = True
                if sum([contains_docx, contains_pptx, contains_xlsx]) > 1:
                    log.debug("Bad [>1 of DOCX, PPTX, XLSX content]: {}",
                              filename)
                    return False

            return True
    except (BadZipFile, OSError) as e:
        # ---------------------------------------------------------------------
        # Duff file. Easy!
        # ---------------------------------------------------------------------
        log.debug("Bad [BadZipFile or OSError]: {!r}; error was {!r}",
                  filename, e)
        return False

Response: Determines whether an OpenXML file appears to be good.


Instruction:
How would you code a function in Python 3 to
process an OpenXML file based on whether it is good or not.

Response:
def process_openxml_file(filename: str,
                         print_good: bool,
                         delete_if_bad: bool) -> None:
    """
    Prints the filename of, or deletes, an OpenXML file depending on whether
    it is corrupt or not.

    Args:
        filename: filename to check
        print_good: if ``True``, then prints the filename if the file
            appears good.
        delete_if_bad: if ``True``, then deletes the file if the file
            appears corrupt.
    """
    print_bad = not print_good
    try:
        file_good = is_openxml_good(filename)
        file_bad = not file_good
        if (print_good and file_good) or (print_bad and file_bad):
            print(filename)
        if delete_if_bad and file_bad:
            log.warning("Deleting: {}", filename)
            os.remove(filename)
    except Exception as e:
        # Must explicitly catch and report errors, since otherwise they vanish
        # into the ether.
        log.critical("Uncaught error in subprocess: {!r}\n{}", e,
                     traceback.format_exc())
        raise


Instruction:
How would you explain what the following Python 3 function does
def main() -> None:
    parser = ArgumentParser(
        formatter_class=RawDescriptionHelpFormatter,
        description="""
Tool to scan rescued Microsoft Office OpenXML files (produced by the
find_recovered_openxml.py tool in this kit; q.v.) and detect bad (corrupted)
ones.
        """
    )
    parser.add_argument(
        "filename", nargs="*",
        help="File(s) to check. You can also specify directores if you use "
             "--recursive"
    )
    parser.add_argument(
        "--filenames_from_stdin", "-x", action="store_true",
        help="Take filenames from stdin instead, one line per filename "
             "(useful for chained grep)."
    )
    parser.add_argument(
        "--recursive", action="store_true",
        help="Allow search to descend recursively into any directories "
             "encountered."
    )
    parser.add_argument(
        "--skip_files", nargs="*", default=[],
        help="File pattern(s) to skip. You can specify wildcards like '*.txt' "
             "(but you will have to enclose that pattern in quotes under "
             "UNIX-like operating systems). The basename of each file will be "
             "tested against these filenames/patterns. Consider including "
             "Scalpel's 'audit.txt'."
    )
    parser.add_argument(
        "--good", action="store_true",
        help="List good files, not bad"
    )
    parser.add_argument(
        "--delete_if_bad", action="store_true",
        help="If a file is found to be bad, delete it. DANGEROUS."
    )
    parser.add_argument(
        "--run_repeatedly", type=int,
        help="Run the tool repeatedly with a pause of <run_repeatedly> "
             "seconds between runs. (For this to work well with the move/"
             "delete options, you should specify one or more DIRECTORIES in "
             "the 'filename' arguments, not files, and you will need the "
             "--recursive option.)"
    )
    parser.add_argument(
        "--nprocesses", type=int, default=multiprocessing.cpu_count(),
        help="Specify the number of processes to run in parallel."
    )
    parser.add_argument(
        "--verbose", action="store_true",
        help="Verbose output"
    )
    args = parser.parse_args()
    main_only_quicksetup_rootlogger(
        level=logging.DEBUG if args.verbose else logging.INFO,
        with_process_id=True
    )
    if bool(args.filenames_from_stdin) == bool(args.filename):
        raise ValueError("Specify --filenames_from_stdin or filenames on the "
                         "command line, but not both")
    if args.filenames_from_stdin and args.run_repeatedly:
        raise ValueError("Can't use both --filenames_from_stdin and "
                         "--run_repeatedly")

    # Repeated scanning loop
    while True:
        log.debug("Starting scan.")
        log.debug("- Scanning files/directories {!r}{}",
                  args.filename,
                  " recursively" if args.recursive else "")
        log.debug("- Skipping files matching {!r}", args.skip_files)
        log.debug("- Using {} simultaneous processes", args.nprocesses)
        log.debug("- Reporting {} filenames", "good" if args.good else "bad")
        if args.delete_if_bad:
            log.warning("- Deleting bad OpenXML files.")

        # Iterate through files
        pool = multiprocessing.Pool(processes=args.nprocesses)

        if args.filenames_from_stdin:
            generator = gen_from_stdin()
        else:
            generator = gen_filenames(starting_filenames=args.filename,
                                      recursive=args.recursive)

        for filename in generator:
            src_basename = os.path.basename(filename)
            if any(fnmatch.fnmatch(src_basename, pattern)
                   for pattern in args.skip_files):
                log.debug("Skipping file as ordered: " + filename)
                continue
            exists, locked = exists_locked(filename)
            if locked or not exists:
                log.debug("Skipping currently inaccessible file: " + filename)
                continue
            kwargs = {
                'filename': filename,
                'print_good': args.good,
                'delete_if_bad': args.delete_if_bad,
            }
            # log.critical("start")
            pool.apply_async(process_openxml_file, [], kwargs)
            # result = pool.apply_async(process_file, [], kwargs)
            # result.get()  # will re-raise any child exceptions
            # ... but it waits for the process to complete! That's no help.
            # log.critical("next")
            # ... https://stackoverflow.com/questions/22094852/how-to-catch-exceptions-in-workers-in-multiprocessing  # noqa
        pool.close()
        pool.join()

        log.debug("Finished scan.")
        if args.run_repeatedly is None:
            break
        log.info("Sleeping for {} s...", args.run_repeatedly)
        sleep(args.run_repeatedly)

Response: This function is called by the command line interface to find bad OpenXML files.


Instruction:
Can you tell what is the following Python 3 function doing
def open(self) -> bool:
        if self.connection:
            # Nothing to do if the connection is already open.
            return False

        connection_params = {'local_hostname': DNS_NAME.get_fqdn()}
        if self.timeout is not None:
            connection_params['timeout'] = self.timeout
        try:
            self.connection = smtplib.SMTP(self.host, self.port,
                                           **connection_params)

            # TLS
            context = ssl.SSLContext(self._protocol())
            if self.ssl_certfile:
                context.load_cert_chain(certfile=self.ssl_certfile,
                                        keyfile=self.ssl_keyfile)
            self.connection.ehlo()
            self.connection.starttls(context=context)
            self.connection.ehlo()
            if self.username and self.password:
                self.connection.login(self.username, self.password)
                log.debug("Successful SMTP connection/login")
            else:
                log.debug("Successful SMTP connection (without login)")
            return True
        except smtplib.SMTPException:
            log.debug("SMTP connection and/or login failed")
            if not self.fail_silently:
                raise

Response: Open the SMTP connection to the email server.


Instruction:
Explain what the following Python 3 code does
def bhattacharyya_distance(pca1,pca2):
    u1 = pca1.coefficients
    s1 = pca1.covariance_matrix
    u2 = pca2.coefficients
    s2 = pca2.covariance_matrix

    sigma = (s1+s2)/2

    assert all(u1 > 0)
    assert all(u2 > 0)
    assert all(s1.sum(axis=1) > 0)
    assert all(s2.sum(axis=1) > 0)

    _ = 1/8*dot((u1-u2).T, N.linalg.inv(sigma),u1-u2)
    _ += 1/2*N.log(N.linalg.det(sigma)/(N.linalg.det(s1)*N.linalg.det(s2)))
    return _

Response: A measure of the distance between two probability distributions


Instruction:
Can you tell what is the following Python 3 function doing
def produce_csv_output(filehandle: TextIO,
                       fields: Sequence[str],
                       values: Iterable[str]) -> None:
    """
    Produce CSV output, without using ``csv.writer``, so the log can be used
    for lots of things.

    - ... eh? What was I talking about?
    - POOR; DEPRECATED.

    Args:
        filehandle: file to write to
        fields: field names
        values: values
    """
    output_csv(filehandle, fields)
    for row in values:
        output_csv(filehandle, row)

Response: Produce CSV output for the current language.


Instruction:
How would you implement a function in Python 3 that
writes a line of CSV. POOR does not escape things properly. DEPRECATED.

Response:
def output_csv(filehandle: TextIO, values: Iterable[str]) -> None:
    """
    Write a line of CSV. POOR; does not escape things properly. DEPRECATED.

    Args:
        filehandle: file to write to
        values: values
    """
    line = ",".join(values)
    filehandle.write(line + "\n")


Instruction:
Can you generate a brief explanation for the following Python 3 code
def get_what_follows_raw(s: str,
                         prefix: str,
                         onlyatstart: bool = True,
                         stripwhitespace: bool = True) -> Tuple[bool, str]:
    """
    Find the part of ``s`` that is after ``prefix``.

    Args:
        s: string to analyse
        prefix: prefix to find
        onlyatstart: only accept the prefix if it is right at the start of
            ``s``
        stripwhitespace: remove whitespace from the result

    Returns:
        tuple: ``(found, result)``

    """
    prefixstart = s.find(prefix)
    if ((prefixstart == 0 and onlyatstart) or
            (prefixstart != -1 and not onlyatstart)):
        # substring found
        resultstart = prefixstart + len(prefix)
        result = s[resultstart:]
        if stripwhitespace:
            result = result.strip()
        return True, result
    return False, ""

Response: Find the part of s that is after prefix.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def get_what_follows(strings: Sequence[str],
                     prefix: str,
                     onlyatstart: bool = True,
                     stripwhitespace: bool = True,
                     precedingline: str = "") -> str:
    """
    Find a string in ``strings`` that begins with ``prefix``; return the part
    that's after ``prefix``. Optionally, require that the preceding string
    (line) is ``precedingline``.

    Args:
        strings: strings to analyse
        prefix: prefix to find
        onlyatstart: only accept the prefix if it is right at the start of
            ``s``
        stripwhitespace: remove whitespace from the result
        precedingline: if truthy, require that the preceding line be as
            specified here

    Returns:
        the line fragment

    """
    if not precedingline:
        for s in strings:
            (found, result) = get_what_follows_raw(s, prefix, onlyatstart,
                                                   stripwhitespace)
            if found:
                return result
        return ""
    else:
        for i in range(1, len(strings)):  # i indexes the second of a pair
            if strings[i-1].find(precedingline) == 0:
                # ... if found at the start
                (found, result) = get_what_follows_raw(strings[i], prefix,
                                                       onlyatstart,
                                                       stripwhitespace)
                if found:
                    return result
        return ""

Response: Returns the string that follows the prefix.


Instruction:
Can you generate the documentation for the following Python 3 function
def get_string(strings: Sequence[str],
               prefix: str,
               ignoreleadingcolon: bool = False,
               precedingline: str = "") -> Optional[str]:
    """
    Find a string as per :func:`get_what_follows`.

    Args:
        strings: see :func:`get_what_follows`
        prefix: see :func:`get_what_follows`
        ignoreleadingcolon: if ``True``, restrict the result to what comes
            after its first colon (and whitespace-strip that)
        precedingline: see :func:`get_what_follows`

    Returns:
        the line fragment

    """
    s = get_what_follows(strings, prefix, precedingline=precedingline)
    if ignoreleadingcolon:
        f = s.find(":")
        if f != -1:
            s = s[f+1:].strip()
    if len(s) == 0:
        return None
    return s

Response: Find a string as per get_what_follows.


Instruction:
Make a summary of the following Python 3 code
def get_string_relative(strings: Sequence[str],
                        prefix1: str,
                        delta: int,
                        prefix2: str,
                        ignoreleadingcolon: bool = False,
                        stripwhitespace: bool = True) -> Optional[str]:
    """
    Finds a line (string) in ``strings`` beginning with ``prefix1``. Moves
    ``delta`` lines (strings) further. Returns the end of the line that
    begins with ``prefix2``, if found.

    Args:
        strings: as above
        prefix1: as above
        delta: as above
        prefix2: as above
        ignoreleadingcolon: restrict the result to the part after its first
            colon?
        stripwhitespace: strip whitespace from the start/end of the result?

    Returns:
        the line fragment
    """
    for firstline in range(0, len(strings)):
        if strings[firstline].find(prefix1) == 0:  # if found...
            secondline = firstline + delta
            if secondline < 0 or secondline >= len(strings):
                continue
            if strings[secondline].find(prefix2) == 0:
                s = strings[secondline][len(prefix2):]
                if stripwhitespace:
                    s = s.strip()
                if ignoreleadingcolon:
                    f = s.find(":")
                    if f != -1:
                        s = s[f+1:].strip()
                    if stripwhitespace:
                        s = s.strip()
                if len(s) == 0:
                    return None
                return s
    return None

Response: Returns the string that is relative to the first line in the sequence of strings.


Instruction:
Can you implement a function in Python 3 that
retrieves an integer parameter from a sequence of strings.

Response:
def get_int(strings: Sequence[str],
            prefix: str,
            ignoreleadingcolon: bool = False,
            precedingline: str = "") -> Optional[int]:
    """
    Fetches an integer parameter via :func:`get_string`.
    """
    return get_int_raw(get_string(strings, prefix,
                                  ignoreleadingcolon=ignoreleadingcolon,
                                  precedingline=precedingline))


Instruction:
Can you tell what is the following Python 3 function doing
def get_float(strings: Sequence[str],
              prefix: str,
              ignoreleadingcolon: bool = False,
              precedingline: str = "") -> Optional[float]:
    """
    Fetches a float parameter via :func:`get_string`.
    """
    return get_float_raw(get_string(strings, prefix,
                                    ignoreleadingcolon=ignoreleadingcolon,
                                    precedingline=precedingline))

Response: Retrieves a float parameter from a sequence of strings.


Instruction:
How would you explain what the following Python 3 function does
def get_bool_raw(s: str) -> Optional[bool]:
    if s == "Y" or s == "y":
        return True
    elif s == "N" or s == "n":
        return False
    return None

Response: Maps Y to True and N to False.


Instruction:
Implement a function in Python 3 to
fetch a boolean parameter from a sequence of strings.

Response:
def get_bool(strings: Sequence[str],
             prefix: str,
             ignoreleadingcolon: bool = False,
             precedingline: str = "") -> Optional[bool]:
    """
    Fetches a boolean parameter via :func:`get_string`.
    """
    return get_bool_raw(get_string(strings, prefix,
                                   ignoreleadingcolon=ignoreleadingcolon,
                                   precedingline=precedingline))


Instruction:
How would you explain what the following Python 3 function does
def get_bool_relative(strings: Sequence[str],
                      prefix1: str,
                      delta: int,
                      prefix2: str,
                      ignoreleadingcolon: bool = False) -> Optional[bool]:
    """
    Fetches a boolean parameter via :func:`get_string_relative`.
    """
    return get_bool_raw(get_string_relative(
        strings, prefix1, delta, prefix2,
        ignoreleadingcolon=ignoreleadingcolon))

Response: Gets a boolean parameter from a sequence of strings relative to a given base.


Instruction:
Can you tell what is the following Python 3 function doing
def get_float_relative(strings: Sequence[str],
                       prefix1: str,
                       delta: int,
                       prefix2: str,
                       ignoreleadingcolon: bool = False) -> Optional[float]:
    """
    Fetches a float parameter via :func:`get_string_relative`.
    """
    return get_float_raw(get_string_relative(
        strings, prefix1, delta, prefix2,
        ignoreleadingcolon=ignoreleadingcolon))

Response: Gets a float parameter from a sequence of strings relative to a given prefix.


Instruction:
Can you write a function in Python 3 where it
gets an integer parameter from a sequence of strings relative to another string.

Response:
def get_int_relative(strings: Sequence[str],
                     prefix1: str,
                     delta: int,
                     prefix2: str,
                     ignoreleadingcolon: bool = False) -> Optional[int]:
    """
    Fetches an int parameter via :func:`get_string_relative`.
    """
    return get_int_raw(get_string_relative(
        strings, prefix1, delta, prefix2,
        ignoreleadingcolon=ignoreleadingcolon))


Instruction:
Can you create a Python 3 function that
retrieves a datetime. datetime parameter from a sequence of strings.

Response:
def get_datetime(strings: Sequence[str],
                 prefix: str,
                 datetime_format_string: str,
                 ignoreleadingcolon: bool = False,
                 precedingline: str = "") -> Optional[datetime.datetime]:
    """
    Fetches a ``datetime.datetime`` parameter via :func:`get_string`.
    """
    x = get_string(strings, prefix, ignoreleadingcolon=ignoreleadingcolon,
                   precedingline=precedingline)
    if len(x) == 0:
        return None
    # For the format strings you can pass to datetime.datetime.strptime, see
    # http://docs.python.org/library/datetime.html
    # A typical one is "%d-%b-%Y (%H:%M:%S)"
    d = datetime.datetime.strptime(x, datetime_format_string)
    return d


Instruction:
Write a Python 3 function that can
find the index of the line in strings that begins with linestart.

Response:
def find_line_beginning(strings: Sequence[str],
                        linestart: Optional[str]) -> int:
    """
    Finds the index of the line in ``strings`` that begins with ``linestart``,
    or ``-1`` if none is found.

    If ``linestart is None``, match an empty line.
    """
    if linestart is None:  # match an empty line
        for i in range(len(strings)):
            if is_empty_string(strings[i]):
                return i
        return -1
    for i in range(len(strings)):
        if strings[i].find(linestart) == 0:
            return i
    return -1


Instruction:
Can you implement a function in Python 3 that
finds the index of the line in strings that contains contents.

Response:
def find_line_containing(strings: Sequence[str], contents: str) -> int:
    """
    Finds the index of the line in ``strings`` that contains ``contents``,
    or ``-1`` if none is found.
    """
    for i in range(len(strings)):
        if strings[i].find(contents) != -1:
            return i
    return -1


Instruction:
Given the following Python 3 function, write the documentation
def get_lines_from_to(strings: List[str],
                      firstlinestart: str,
                      list_of_lastline_starts: Iterable[Optional[str]]) \
        -> List[str]:
    """
    Takes a list of ``strings``. Returns a list of strings FROM
    ``firstlinestart`` (inclusive) TO the first of ``list_of_lastline_starts``
    (exclusive).

    To search to the end of the list, use ``list_of_lastline_starts = []``.

    To search to a blank line, use ``list_of_lastline_starts = [None]``
    """
    start_index = find_line_beginning(strings, firstlinestart)
    # log.debug("start_index: {}", start_index)
    if start_index == -1:
        return []
    end_offset = None  # itself a valid slice index
    for lls in list_of_lastline_starts:
        possible_end_offset = find_line_beginning(strings[start_index:], lls)
        # log.debug("lls {!r} -> possible_end_offset {}",
        #           lls, possible_end_offset)
        if possible_end_offset != -1:  # found one
            if end_offset is None or possible_end_offset < end_offset:
                end_offset = possible_end_offset
    end_index = None if end_offset is None else (start_index + end_offset)
    # log.debug("end_index: {}", end_index)
    return strings[start_index:end_index]

Response: Takes a list of strings and returns a list of strings FROM firstlinestart TO lastline_starts.


Instruction:
Write a Python 3 script to
return a list of fields from a CSV file.

Response:
def csv_to_list_of_fields(lines: List[str],
                          csvheader: str,
                          quotechar: str = '"') -> List[str]:
    """
    Extracts data from a list of CSV lines (starting with a defined header
    line) embedded in a longer text block but ending with a blank line.

    Used for processing e.g. MonkeyCantab rescue text output.

    Args:
        lines: CSV lines
        csvheader: CSV header line
        quotechar: ``quotechar`` parameter passed to :func:`csv.reader`

    Returns:
        list (by row) of lists (by value); see example

    Test code:

    .. code-block:: python

        import logging
        from cardinal_pythonlib.rnc_text import *
        logging.basicConfig(level=logging.DEBUG)

        myheader = "field1,field2,field3"
        mycsvlines = [
            "irrelevant line",
            myheader,  # header: START
            "row1value1,row1value2,row1value3",
            "row2value1,row2value2,row2value3",
            "",  # terminating blank line: END
            "other irrelevant line",
        ]
        csv_to_list_of_fields(mycsvlines, myheader)
        # [['row1value1', 'row1value2', 'row1value3'], ['row2value1', 'row2value2', 'row2value3']]

    """  # noqa
    data = []  # type: List[str]
    # an empty line marks the end of the block
    csvlines = get_lines_from_to(lines, csvheader, [None])[1:]
    # ... remove the CSV header
    reader = csv.reader(csvlines, quotechar=quotechar)
    for fields in reader:
        data.append(fields)
    return data


Instruction:
Can you generate the documentation for the following Python 3 function
def csv_to_list_of_dicts(lines: List[str],
                         csvheader: str,
                         quotechar: str = '"') -> List[Dict[str, str]]:
    """
    Extracts data from a list of CSV lines (starting with a defined header
    line) embedded in a longer text block but ending with a blank line.

    Args:
        lines: CSV lines
        csvheader: CSV header line
        quotechar: ``quotechar`` parameter passed to :func:`csv.reader`

    Returns:
        list of dictionaries mapping fieldnames (from the header) to values

    """
    data = []  # type: List[Dict[str, str]]
    # an empty line marks the end of the block
    csvlines = get_lines_from_to(lines, csvheader, [None])[1:]
    # ... remove the CSV header
    headerfields = csvheader.split(",")
    reader = csv.reader(csvlines, quotechar=quotechar)
    for fields in reader:
        row = {}  # type: Dict[str, str]
        for f in range(len(headerfields)):
            row[headerfields[f]] = fields[f]
        data.append(row)
    return data

Response: Extracts data from a list of CSV lines starting with a defined header line and ending with a blank line.


Instruction:
Can you generate the documentation for the following Python 3 function
def dictlist_convert_to_string(dict_list: Iterable[Dict], key: str) -> None:
    for d in dict_list:
        d[key] = str(d[key])
        if d[key] == "":
            d[key] = None

Response: Converts a list of dictionaries to a string form.


Instruction:
Explain what the following Python 3 code does
def dictlist_convert_to_datetime(dict_list: Iterable[Dict],
                                 key: str,
                                 datetime_format_string: str) -> None:
    """
    Process an iterable of dictionaries. For each dictionary ``d``, convert
    (in place) ``d[key]`` to a ``datetime.datetime`` form, using
    ``datetime_format_string`` as the format parameter to
    :func:`datetime.datetime.strptime`.
    """
    for d in dict_list:
        d[key] = datetime.datetime.strptime(d[key], datetime_format_string)

Response: Converts a list of dictionaries into a datetime. datetime object.


Instruction:
Can you generate the documentation for the following Python 3 function
def dictlist_convert_to_int(dict_list: Iterable[Dict], key: str) -> None:
    for d in dict_list:
        try:
            d[key] = int(d[key])
        except ValueError:
            d[key] = None

Response: Converts each dictionary d [ key ] to an integer.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def dictlist_convert_to_float(dict_list: Iterable[Dict], key: str) -> None:
    for d in dict_list:
        try:
            d[key] = float(d[key])
        except ValueError:
            d[key] = None

Response: Converts each dictionary in dict_list to a float.


Instruction:
Can you generate the documentation for the following Python 3 function
def dictlist_convert_to_bool(dict_list: Iterable[Dict], key: str) -> None:
    for d in dict_list:
        # d[key] = True if d[key] == "Y" else False
        d[key] = 1 if d[key] == "Y" else 0

Response: Convert a list of dictionaries to a bool.


Instruction:
Implement a Python 3 function for
replacing a key with value in a list of dictionaries.

Response:
def dictlist_replace(dict_list: Iterable[Dict], key: str, value: Any) -> None:
    """
    Process an iterable of dictionaries. For each dictionary ``d``, change
    (in place) ``d[key]`` to ``value``.
    """
    for d in dict_list:
        d[key] = value


Instruction:
Write a Python 3 script to
remove a key from a list of dictionaries.

Response:
def dictlist_wipe_key(dict_list: Iterable[Dict], key: str) -> None:
    """
    Process an iterable of dictionaries. For each dictionary ``d``, delete
    ``d[key]`` if it exists.
    """
    for d in dict_list:
        d.pop(key, None)