Instruction:
Can you generate the documentation for the following Python 3 function
def add_pending_model_content(cursor, publication_id, model):
    cursor.execute("""\
        SELECT id, ident_hash(uuid, major_version, minor_version)
        FROM pending_documents
        WHERE publication_id = %s AND uuid = %s""",
                   (publication_id, model.id,))
    document_info = cursor.fetchone()

    def attach_info_to_exception(exc):
        """Small cached function to grab the pending document id
        and hash to attach to the exception, which is useful when
        reading the json data on a response.
        """
        exc.publication_id = publication_id
        exc.pending_document_id, exc.pending_ident_hash = document_info

    def mark_invalid_reference(reference):
        """Set the publication to failure and attach invalid reference
        to the publication.
        """
        exc = exceptions.InvalidReference(reference)
        attach_info_to_exception(exc)
        set_publication_failure(cursor, exc)

    for resource in getattr(model, 'resources', []):
        add_pending_resource(cursor, resource, document=model)

    if isinstance(model, cnxepub.Document):
        for reference in model.references:
            if reference.is_bound:
                reference.bind(reference.bound_model, '/resources/{}')
            elif reference.remote_type == cnxepub.INTERNAL_REFERENCE_TYPE:
                if reference.uri.startswith('#'):
                    pass
                elif reference.uri.startswith('/contents'):
                    ident_hash = parse_archive_uri(reference.uri)
                    try:
                        doc_pointer = lookup_document_pointer(
                            ident_hash, cursor)
                    except DocumentLookupError:
                        mark_invalid_reference(reference)
                    else:
                        reference.bind(doc_pointer, "/contents/{}")
                else:
                    mark_invalid_reference(reference)
            # else, it's a remote or cnx.org reference ...Do nothing.

        args = (psycopg2.Binary(model.content.encode('utf-8')),
                publication_id, model.id,)
        stmt = """\
            UPDATE "pending_documents"
            SET ("content") = (%s)
            WHERE "publication_id" = %s AND "uuid" = %s"""
    else:
        metadata = model.metadata.copy()
        # All document pointers in the tree are valid?
        document_pointers = [m for m in cnxepub.flatten_model(model)
                             if isinstance(m, cnxepub.DocumentPointer)]
        document_pointer_ident_hashes = [
            (split_ident_hash(dp.ident_hash)[0],
             split_ident_hash(dp.ident_hash, split_version=True)[1][0],
             split_ident_hash(dp.ident_hash, split_version=True)[1][1],)
            #  split_ident_hash(dp.ident_hash, split_version=True)[1][0],)
            for dp in document_pointers]
        document_pointer_ident_hashes = zip(*document_pointer_ident_hashes)

        if document_pointers:
            uuids, major_vers, minor_vers = document_pointer_ident_hashes
            cursor.execute("""\
SELECT dp.uuid, module_version(dp.maj_ver, dp.min_ver) AS version,
       dp.uuid = m.uuid AS exists,
       m.portal_type = 'Module' AS is_document
FROM (SELECT unnest(%s::uuid[]), unnest(%s::integer[]), unnest(%s::integer[]))\
         AS dp(uuid, maj_ver, min_ver)
     LEFT JOIN modules AS m ON dp.uuid = m.uuid AND \
         (dp.maj_ver = m.major_version OR dp.maj_ver is null)""",
                           (list(uuids), list(major_vers), list(minor_vers),))
            valid_pointer_results = cursor.fetchall()
            for result_row in valid_pointer_results:
                uuid, version, exists, is_document = result_row
                if not (exists and is_document):
                    dp = [dp for dp in document_pointers
                          if dp.ident_hash == join_ident_hash(uuid, version)
                          ][0]
                    exc = exceptions.InvalidDocumentPointer(
                        dp, exists=exists, is_document=is_document)
                    attach_info_to_exception(exc)
                    set_publication_failure(cursor, exc)

        # Insert the tree into the metadata.
        metadata['_tree'] = cnxepub.model_to_tree(model)
        args = (json.dumps(metadata),
                None,  # TODO Render the HTML tree at ``model.content``.
                publication_id, model.id,)
        # Must pave over metadata because postgresql lacks built-in
        # json update functions.
        stmt = """\
            UPDATE "pending_documents"
            SET ("metadata", "content") = (%s, %s)
            WHERE "publication_id" = %s AND "uuid" = %s"""
    cursor.execute(stmt, args)

Response: Adds the pending model s content to the pending model s content.


Instruction:
Can you generate the documentation for the following Python 3 function
def set_publication_failure(cursor, exc):
    publication_id = exc.publication_id
    if publication_id is None:
        raise ValueError("Exception must have a ``publication_id`` value.")
    cursor.execute("""\
SELECT "state_messages"
FROM publications
WHERE id = %s""", (publication_id,))
    state_messages = cursor.fetchone()[0]
    if state_messages is None:
        state_messages = []
    entry = exc.__dict__
    entry['message'] = exc.message
    state_messages.append(entry)
    state_messages = json.dumps(state_messages)
    cursor.execute("""\
UPDATE publications SET ("state", "state_messages") = (%s, %s)
WHERE id = %s""", ('Failed/Error', state_messages, publication_id,))

Response: Sets the publication as failed and append the failure message to the publication record.


Instruction:
Create a Python 3 function for
adding a publication entry and makes each item a pending document.

Response:
def add_publication(cursor, epub, epub_file, is_pre_publication=False):
    """Adds a publication entry and makes each item
    a pending document.
    """
    publisher = epub[0].metadata['publisher']
    publish_message = epub[0].metadata['publication_message']
    epub_binary = psycopg2.Binary(epub_file.read())
    args = (publisher, publish_message, epub_binary, is_pre_publication,)
    cursor.execute("""\
INSERT INTO publications
  ("publisher", "publication_message", "epub", "is_pre_publication")
VALUES (%s, %s, %s, %s)
RETURNING id
""", args)
    publication_id = cursor.fetchone()[0]
    insert_mapping = {}

    models = set([])
    for package in epub:
        binder = cnxepub.adapt_package(package)
        if binder in models:
            continue
        for document in cnxepub.flatten_to_documents(binder):
            if document not in models:
                ident_hash = add_pending_model(
                    cursor, publication_id, document)
                insert_mapping[document.id] = ident_hash
                models.add(document)
        # The binding object could be translucent/see-through,
        # (case for a binder that only contains loose-documents).
        # Otherwise we should also publish the the binder.
        if not binder.is_translucent:
            ident_hash = add_pending_model(cursor, publication_id, binder)
            insert_mapping[binder.id] = ident_hash
            models.add(binder)
    for model in models:
        # Now that all models have been given an identifier
        # we can write the content to the database.
        try:
            add_pending_model_content(cursor, publication_id, model)
        except ResourceFileExceededLimitError as e:
            e.publication_id = publication_id
            set_publication_failure(cursor, e)
    return publication_id, insert_mapping


Instruction:
Here you have a function in Python 3, explain what it does
def _check_pending_document_license_state(cursor, document_id):
    cursor.execute("""\
SELECT BOOL_AND(accepted IS TRUE)
FROM
  pending_documents AS pd,
  license_acceptances AS la
WHERE
  pd.id = %s
  AND
  pd.uuid = la.uuid""",
                   (document_id,))
    try:
        is_accepted = cursor.fetchone()[0]
    except TypeError:
        # There are no licenses associated with this document.
        is_accepted = True
    return is_accepted

Response: Check the aggregate state on the pending document.


Instruction:
Can you generate the documentation for the following Python 3 function
def _check_pending_document_role_state(cursor, document_id):
    cursor.execute("""\
SELECT BOOL_AND(accepted IS TRUE)
FROM
  role_acceptances AS ra,
  pending_documents as pd
WHERE
  pd.id = %s
  AND
  pd.uuid = ra.uuid""",
                   (document_id,))
    try:
        is_accepted = cursor.fetchone()[0]
    except TypeError:
        # There are no roles to accept
        is_accepted = True
    return is_accepted

Response: Check the aggregate state on the pending document.


Instruction:
Create a Python 3 function for
updating the state of the pending document.

Response:
def _update_pending_document_state(cursor, document_id, is_license_accepted,
                                   are_roles_accepted):
    """Update the state of the document's state values."""
    args = (bool(is_license_accepted), bool(are_roles_accepted),
            document_id,)
    cursor.execute("""\
UPDATE pending_documents
SET (license_accepted, roles_accepted) = (%s, %s)
WHERE id = %s""",
                   args)


Instruction:
Write a Python 3 function that can
check to see if the publication contains any revised models.

Response:
def is_revision_publication(publication_id, cursor):
    """Checks to see if the publication contains any revised models.
    Revised in this context means that it is a new version of an
    existing piece of content.
    """
    cursor.execute("""\
SELECT 't'::boolean FROM modules
WHERE uuid IN (SELECT uuid
               FROM pending_documents
               WHERE publication_id = %s)
LIMIT 1""", (publication_id,))
    try:
        cursor.fetchone()[0]
    except TypeError:  # NoneType
        has_revision_models = False
    else:
        has_revision_models = True
    return has_revision_models


Instruction:
How would you implement a function in Python 3 that
invokes to update and acquire its current state. This is used to update the publication to archive.

Response:
def poke_publication_state(publication_id, cursor):
    """Invoked to poke at the publication to update and acquire its current
    state. This is used to persist the publication to archive.
    """
    cursor.execute("""\
SELECT "state", "state_messages", "is_pre_publication", "publisher"
FROM publications
WHERE id = %s""", (publication_id,))
    row = cursor.fetchone()
    current_state, messages, is_pre_publication, publisher = row

    if current_state in END_N_INTERIM_STATES:
        # Bailout early, because the publication is either in progress
        # or has been completed.
        return current_state, messages

    # Check for acceptance...
    cursor.execute("""\
SELECT
  pd.id, license_accepted, roles_accepted
FROM publications AS p JOIN pending_documents AS pd ON p.id = pd.publication_id
WHERE p.id = %s
""", (publication_id,))
    pending_document_states = cursor.fetchall()
    publication_state_mapping = {}
    for document_state in pending_document_states:
        id, is_license_accepted, are_roles_accepted = document_state
        publication_state_mapping[id] = [is_license_accepted,
                                         are_roles_accepted]
        has_changed_state = False
        if is_license_accepted and are_roles_accepted:
            continue
        if not is_license_accepted:
            accepted = _check_pending_document_license_state(
                cursor, id)
            if accepted != is_license_accepted:
                has_changed_state = True
                is_license_accepted = accepted
                publication_state_mapping[id][0] = accepted
        if not are_roles_accepted:
            accepted = _check_pending_document_role_state(
                cursor, id)
            if accepted != are_roles_accepted:
                has_changed_state = True
                are_roles_accepted = accepted
                publication_state_mapping[id][1] = accepted
        if has_changed_state:
            _update_pending_document_state(cursor, id,
                                           is_license_accepted,
                                           are_roles_accepted)

    # Are all the documents ready for publication?
    state_lump = set([l and r for l, r in publication_state_mapping.values()])
    is_publish_ready = not (False in state_lump) and not (None in state_lump)
    change_state = "Done/Success"
    if not is_publish_ready:
        change_state = "Waiting for acceptance"

    # Does this publication need moderation? (ignore on pre-publication)
    # TODO Is this a revision publication? If so, it doesn't matter who the
    #      user is, because they have been vetted by the previous publisher.
    #      This has loopholes...
    if not is_pre_publication and is_publish_ready:
        # Has this publisher been moderated before?
        cursor.execute("""\
SELECT is_moderated
FROM users AS u LEFT JOIN publications AS p ON (u.username = p.publisher)
WHERE p.id = %s""",
                       (publication_id,))
        try:
            is_publisher_moderated = cursor.fetchone()[0]
        except TypeError:
            is_publisher_moderated = False

        # Are any of these documents a revision? Thus vetting of
        #   the publisher was done by a vetted peer.
        if not is_publisher_moderated \
           and not is_revision_publication(publication_id, cursor):
            # Hold up! This publish needs moderation.
            change_state = "Waiting for moderation"
            is_publish_ready = False

    # Publish the pending documents.
    if is_publish_ready:
        change_state = "Done/Success"
        if not is_pre_publication:
            publication_state = publish_pending(cursor, publication_id)
        else:
            cursor.execute("""\
UPDATE publications
SET state = %s
WHERE id = %s
RETURNING state, state_messages""", (change_state, publication_id,))
            publication_state, messages = cursor.fetchone()
    else:
        # `change_state` set prior to this...
        cursor.execute("""\
UPDATE publications
SET state = %s
WHERE id = %s
RETURNING state, state_messages""", (change_state, publication_id,))
        publication_state, messages = cursor.fetchone()

    return publication_state, messages


Instruction:
Can you write a function in Python 3 where it
checks the publication s current state.

Response:
def check_publication_state(publication_id):
    """Check the publication's current state."""
    with db_connect() as db_conn:
        with db_conn.cursor() as cursor:
            cursor.execute("""\
SELECT "state", "state_messages"
FROM publications
WHERE id = %s""", (publication_id,))
            publication_state, publication_messages = cursor.fetchone()
    return publication_state, publication_messages


Instruction:
Create a Python 3 function to
give a tree parse to a set of models

Response:
def _node_to_model(tree_or_item, metadata=None, parent=None,
                   lucent_id=cnxepub.TRANSLUCENT_BINDER_ID):
    """Given a tree, parse to a set of models"""
    if 'contents' in tree_or_item:
        # It is a binder.
        tree = tree_or_item
        binder = cnxepub.TranslucentBinder(metadata=tree)
        for item in tree['contents']:
            node = _node_to_model(item, parent=binder,
                                  lucent_id=lucent_id)
            if node.metadata['title'] != item['title']:
                binder.set_title_for_node(node, item['title'])
        result = binder
    else:
        # It is an item pointing at a document.
        item = tree_or_item
        result = cnxepub.DocumentPointer(item['id'], metadata=item)
    if parent is not None:
        parent.append(result)
    return result


Instruction:
How would you explain what the following Python 3 function does
def _reassemble_binder(id, tree, metadata):
    binder = cnxepub.Binder(id, metadata=metadata)
    for item in tree['contents']:
        node = _node_to_model(item, parent=binder)
        if node.metadata['title'] != item['title']:
            binder.set_title_for_node(node, item['title'])
    return binder

Response: Reassemble a Binder object coming out of the database.


Instruction:
Write a Python 3 script to
write the pending documents to the *Connexions Archive*.

Response:
def publish_pending(cursor, publication_id):
    """Given a publication id as ``publication_id``,
    write the documents to the *Connexions Archive*.
    """
    cursor.execute("""\
WITH state_update AS (
  UPDATE publications SET state = 'Publishing' WHERE id = %s
)
SELECT publisher, publication_message
FROM publications
WHERE id = %s""",
                   (publication_id, publication_id,))
    publisher, message = cursor.fetchone()
    cursor.connection.commit()

    all_models = []

    from .publish import publish_model
    # Commit documents one at a time...
    type_ = cnxepub.Document.__name__
    cursor.execute("""\
SELECT id, uuid, major_version, minor_version, metadata, content
FROM pending_documents
WHERE type = %s AND publication_id = %s""", (type_, publication_id,))
    for row in cursor.fetchall():
        # FIXME Oof, this is hideous!
        id, major_version, minor_version = row[1:4]
        id = str(id)
        version = '.'.join([str(x)
                            for x in (major_version, minor_version,)
                            if x is not None])
        metadata, content = row[-2:]
        content = content[:]
        metadata['version'] = version

        document = cnxepub.Document(id, content, metadata)
        for ref in document.references:
            if ref.uri.startswith('/resources/'):
                hash = ref.uri[len('/resources/'):]
                cursor.execute("""\
SELECT data, media_type
FROM pending_resources
WHERE hash = %s""", (hash,))
                data, media_type = cursor.fetchone()
                document.resources.append(cnxepub.Resource(
                    hash, io.BytesIO(data[:]), media_type, filename=hash))

        _ident_hash = publish_model(cursor, document, publisher, message)  # noqa
        all_models.append(document)

    # And now the binders, one at a time...
    type_ = cnxepub.Binder.__name__
    cursor.execute("""\
SELECT id, uuid, major_version, minor_version, metadata, content
FROM pending_documents
WHERE type = %s AND publication_id = %s""", (type_, publication_id,))
    for row in cursor.fetchall():
        id, major_version, minor_version, metadata = row[1:5]
        tree = metadata['_tree']
        binder = _reassemble_binder(str(id), tree, metadata)
        # Add the resources
        cursor.execute("""\
SELECT hash, data, media_type, filename
FROM pending_resources r
JOIN pending_resource_associations a ON a.resource_id = r.id
JOIN pending_documents d ON a.document_id = d.id
WHERE ident_hash(uuid, major_version, minor_version) = %s""",
                       (binder.ident_hash,))
        binder.resources = [
            cnxepub.Resource(r_hash,
                             io.BytesIO(r_data[:]),
                             r_media_type,
                             filename=r_filename)
            for (r_hash, r_data, r_media_type, r_filename)
            in cursor.fetchall()]
        _ident_hash = publish_model(cursor, binder, publisher, message)  # noqa
        all_models.append(binder)

    # Republish binders containing shared documents.
    from .publish import republish_binders
    _republished_ident_hashes = republish_binders(cursor, all_models)  # noqa

    # Lastly, update the publication status.
    cursor.execute("""\
UPDATE publications
SET state = 'Done/Success'
WHERE id = %s
RETURNING state""", (publication_id,))
    state = cursor.fetchone()[0]
    return state


Instruction:
How would you implement a function in Python 3 that
accepts or deny the document license for the publication_id and user_id.

Response:
def accept_publication_license(cursor, publication_id, user_id,
                               document_ids, is_accepted=False):
    """Accept or deny  the document license for the publication
    (``publication_id``) and user (at ``user_id``)
    for the documents (listed by id as ``document_ids``).
    """
    cursor.execute("""\
UPDATE license_acceptances AS la
SET accepted = %s
FROM pending_documents AS pd
WHERE
  pd.publication_id = %s
  AND
  la.user_id = %s
  AND
  pd.uuid = ANY(%s::UUID[])
  AND
  pd.uuid = la.uuid""",
                   (is_accepted, publication_id, user_id, document_ids,))


Instruction:
Write a Python 3 script to
accept or deny the document role attribution for the publication_id and user_id.

Response:
def accept_publication_role(cursor, publication_id, user_id,
                            document_ids, is_accepted=False):
    """Accept or deny  the document role attribution for the publication
    (``publication_id``) and user (at ``user_id``)
    for the documents (listed by id as ``document_ids``).
    """
    cursor.execute("""\
UPDATE role_acceptances AS ra
SET accepted = %s
FROM pending_documents AS pd
WHERE
  pd.publication_id = %s
  AND
  ra.user_id = %s
  AND
  pd.uuid = ANY(%s::UUID[])
  AND
  pd.uuid = ra.uuid""",
                   (is_accepted, publication_id, user_id, document_ids,))


Instruction:
Can you generate a brief explanation for the following Python 3 code
def upsert_license_requests(cursor, uuid_, roles):
    if not isinstance(roles, (list, set, tuple,)):
        raise TypeError("``roles`` is an invalid type: {}".format(type(roles)))

    acceptors = set([x['uid'] for x in roles])

    # Acquire a list of existing acceptors.
    cursor.execute("""\
SELECT user_id, accepted FROM license_acceptances WHERE uuid = %s""",
                   (uuid_,))
    existing_acceptors = cursor.fetchall()

    # Who's not in the existing list?
    new_acceptors = acceptors.difference([x[0] for x in existing_acceptors])

    # Insert the new licensor acceptors.
    if new_acceptors:
        args = []
        values_fmt = []
        for uid in new_acceptors:
            has_accepted = [x.get('has_accepted', None)
                            for x in roles
                            if uid == x['uid']][0]
            args.extend([uuid_, uid, has_accepted])
            values_fmt.append("(%s, %s, %s)")
        values_fmt = ', '.join(values_fmt)
        cursor.execute("""\
INSERT INTO license_acceptances (uuid, user_id, accepted)
VALUES {}""".format(values_fmt), args)

    # Update any existing license acceptors
    acceptors = set([
        (x['uid'], x.get('has_accepted', None),)
        for x in roles
        # Prevent updating newly inserted records.
        if (x['uid'], x.get('has_accepted', None),) not in new_acceptors
    ])
    existing_acceptors = set([
        x for x in existing_acceptors
        # Prevent updating newly inserted records.
        if x[0] not in new_acceptors
    ])
    tobe_updated_acceptors = acceptors.difference(existing_acceptors)

    for uid, has_accepted in tobe_updated_acceptors:
        cursor.execute("""\
UPDATE license_acceptances SET accepted = %s
WHERE uuid = %s AND user_id = %s""", (has_accepted, uuid_, uid,))

Response: Insert or update license requests for a given uuid and list of roles.


Instruction:
Can you implement a function in Python 3 that
gives a uuid and list of uids remove the identified users s license acceptance entries.

Response:
def remove_license_requests(cursor, uuid_, uids):
    """Given a ``uuid`` and list of ``uids`` (user identifiers)
    remove the identified users' license acceptance entries.
    """
    if not isinstance(uids, (list, set, tuple,)):
        raise TypeError("``uids`` is an invalid type: {}".format(type(uids)))

    acceptors = list(set(uids))

    # Remove the the entries.
    cursor.execute("""\
DELETE FROM license_acceptances
WHERE uuid = %s AND user_id = ANY(%s::text[])""", (uuid_, acceptors,))


Instruction:
Can you create a Python 3 function that
inserts or update a list of role requests.

Response:
def upsert_role_requests(cursor, uuid_, roles):
    """Given a ``uuid`` and list of dicts containing the ``uid`` and
    ``role`` for creating a role acceptance entry. The ``roles`` dict
    can optionally contain a ``has_accepted`` value, which will default
    to true.
    """
    if not isinstance(roles, (list, set, tuple,)):
        raise TypeError("``roles`` is an invalid type: {}"
                        .format(type(roles)))

    acceptors = set([(x['uid'], x['role'],) for x in roles])

    # Acquire a list of existing acceptors.
    cursor.execute("""\
SELECT user_id, role_type, accepted
FROM role_acceptances
WHERE uuid = %s""", (uuid_,))
    existing_roles = cursor.fetchall()

    # Who's not in the existing list?
    existing_acceptors = set([(r, t,) for r, t, _ in existing_roles])
    new_acceptors = acceptors.difference(existing_acceptors)

    # Insert the new role acceptors.
    for acceptor, type_ in new_acceptors:
        has_accepted = [x.get('has_accepted', None)
                        for x in roles
                        if acceptor == x['uid'] and type_ == x['role']][0]
        cursor.execute("""\
INSERT INTO role_acceptances ("uuid", "user_id", "role_type", "accepted")
VALUES (%s, %s, %s, %s)""", (uuid_, acceptor, type_, has_accepted,))

    # Update any existing license acceptors
    acceptors = set([
        (x['uid'], x['role'], x.get('has_accepted', None),)
        for x in roles
        # Prevent updating newly inserted records.
        if (x['uid'], x.get('has_accepted', None),) not in new_acceptors
    ])
    existing_acceptors = set([
        x for x in existing_roles
        # Prevent updating newly inserted records.
        if (x[0], x[1],) not in new_acceptors
    ])
    tobe_updated_acceptors = acceptors.difference(existing_acceptors)

    for uid, type_, has_accepted in tobe_updated_acceptors:
        cursor.execute("""\
UPDATE role_acceptances SET accepted = %s
WHERE uuid = %s AND user_id = %s AND role_type = %s""",
                       (has_accepted, uuid_, uid, type_,))


Instruction:
Create a Python 3 function for
removing the role requests for the given user identifiers and role.

Response:
def remove_role_requests(cursor, uuid_, roles):
    """Given a ``uuid`` and list of dicts containing the ``uid``
    (user identifiers) and ``role`` for removal of the identified
    users' role acceptance entries.
    """
    if not isinstance(roles, (list, set, tuple,)):
        raise TypeError("``roles`` is an invalid type: {}".format(type(roles)))

    acceptors = set([(x['uid'], x['role'],) for x in roles])

    # Remove the the entries.
    for uid, role_type in acceptors:
        cursor.execute("""\
DELETE FROM role_acceptances
WHERE uuid = %s AND user_id = %s AND role_type = %s""",
                       (uuid_, uid, role_type,))


Instruction:
Here you have a function in Python 3, explain what it does
def upsert_acl(cursor, uuid_, permissions):
    if not isinstance(permissions, (list, set, tuple,)):
        raise TypeError("``permissions`` is an invalid type: {}"
                        .format(type(permissions)))

    permissions = set(permissions)

    # Acquire the existin ACL.
    cursor.execute("""\
SELECT user_id, permission
FROM document_acl
WHERE uuid = %s""", (uuid_,))
    existing = set([(r, t,) for r, t in cursor.fetchall()])

    # Who's not in the existing list?
    new_entries = permissions.difference(existing)

    # Insert the new permissions.
    for uid, permission in new_entries:
        cursor.execute("""\
INSERT INTO document_acl
  ("uuid", "user_id", "permission")
VALUES (%s, %s, %s)""", (uuid_, uid, permission))

Response:
Given a uuid and a set of permissions given as a
    tuple of uid and permission upsert them into the database.


Instruction:
Write a Python 3 function for
giving a uuid and a set of permissions remove these entries from the database.

Response:
def remove_acl(cursor, uuid_, permissions):
    """Given a ``uuid`` and a set of permissions given as a tuple
    of ``uid`` and ``permission``, remove these entries from the database.
    """
    if not isinstance(permissions, (list, set, tuple,)):
        raise TypeError("``permissions`` is an invalid type: {}"
                        .format(type(permissions)))

    permissions = set(permissions)

    # Remove the the entries.
    for uid, permission in permissions:
        cursor.execute("""\
DELETE FROM document_acl
WHERE uuid = %s AND user_id = %s AND permission = %s""",
                       (uuid_, uid, permission,))


Instruction:
Explain what the following Python 3 code does
def upsert_users(cursor, user_ids):
    accounts = get_current_registry().getUtility(IOpenstaxAccounts)

    def lookup_profile(username):
        profile = accounts.get_profile_by_username(username)
        # See structure documentation at:
        #   https://<accounts-instance>/api/docs/v1/users/index
        if profile is None:
            raise UserFetchError(username)

        opt_attrs = ('first_name', 'last_name', 'full_name',
                     'title', 'suffix',)
        for attr in opt_attrs:
            profile.setdefault(attr, None)
        return profile

    _upsert_users(cursor, user_ids, lookup_profile)
    _upsert_persons(cursor, user_ids, lookup_profile)

Response: Given a set of user identifiers and a cursor that returns a new user record in the database.


Instruction:
Can you tell what is the following Python 3 function doing
def notify_users(cursor, document_id):
    return

    registry = get_current_registry()
    accounts = registry.getUtility(IOpenstaxAccounts)
    cursor.execute("""\
SELECT la.user_id
FROM license_acceptances AS la
WHERE
  la.uuid = (SELECT uuid FROM pending_documents WHERE id = %s)
  AND la.notified IS NULL AND (NOT la.accepted or la.accepted IS UNKNOWN)
""", (document_id,))
    licensors = [x[0] for x in cursor.fetchall()]

    cursor.execute("""\
SELECT user_id, array_agg(role_type)::text[]
FROM role_acceptances AS ra
WHERE
  ra.uuid = (SELECT uuid FROM pending_documents WHERE id = %s)
  AND ra.notified IS NULL AND (NOT ra.accepted or ra.accepted IS UNKNOWN)
GROUP BY user_id
""", (document_id,))
    roles = {u: r for u, r in cursor.fetchall()}

    needs_notified = set(licensors + roles.keys())

    for user_id in needs_notified:
        data = {
            'user_id': user_id,
            'full_name': None,  # TODO
            'licensor': user_id in licensors,
            'roles': roles.get(user_id, []),
        }
        message = NOTIFICATION_TEMPLATE.render(**data)
        accounts.send_message(user_id, NOFIFICATION_SUBJECT, message)

    cursor.execute("""\
UPDATE license_acceptances SET notified = CURRENT_TIMESTAMP
WHERE
  uuid = (SELECT uuid FROM pending_documents WHERE id = %s)
  AND user_id = ANY (%s)""", (document_id, licensors,))
    # FIXME overwrites notified for all roles types a user might have.
    cursor.execute("""\
UPDATE role_acceptances SET notified = CURRENT_TIMESTAMP
WHERE
  uuid = (SELECT uuid FROM pending_documents WHERE id = %s)
  AND user_id = ANY (%s)""", (document_id, roles.keys(),))

Response: Notify all users about their role and license acceptance for a piece of content associated with the given document_id.


Instruction:
Here you have a function in Python 3, explain what it does
def set_post_publications_state(cursor, module_ident, state_name,
                                state_message=''):  # pragma: no cover
    """This sets the post-publication state in the database."""
    cursor.execute("""\
INSERT INTO post_publications
  (module_ident, state, state_message)
  VALUES (%s, %s, %s)""", (module_ident, state_name, state_message))

Response: This function sets the post - publication state in the database.


Instruction:
Given the following Python 3 function, write the documentation
def get_moderation(request):
    with db_connect() as db_conn:
        with db_conn.cursor() as cursor:
            cursor.execute("""\
SELECT row_to_json(combined_rows) FROM (
  SELECT id, created, publisher, publication_message,
         (select array_agg(row_to_json(pd))
          from pending_documents as pd
          where pd.publication_id = p.id) AS models
  FROM publications AS p
  WHERE state = 'Waiting for moderation') AS combined_rows""")
            moderations = [x[0] for x in cursor.fetchall()]

    return moderations

Response: Return the list of publications that need moderation.


Instruction:
Given the following Python 3 function, write the documentation
def includeme(config):
    settings = config.registry.settings
    session_factory = SignedCookieSessionFactory(settings['session_key'])
    config.set_session_factory(session_factory)

Response: Configures the session manager


Instruction:
Make a summary of the following Python 3 code
def admin_print_styles(request):
    styles = []
    # This fetches all recipes that have been used to successfully bake a
    # current book plus all default recipes that have not yet been used
    # as well as "bad" books that are not "current" state, but would otherwise
    # be the latest/current for that book
    with db_connect(cursor_factory=DictCursor) as db_conn:
        with db_conn.cursor() as cursor:
            cursor.execute("""\
                WITH latest AS (SELECT print_style, recipe,
                    count(*), count(nullif(stateid, 1)) as bad
                FROM modules m
                WHERE portal_type = 'Collection'
                      AND recipe IS NOT NULL
                      AND (
                          baked IS NOT NULL OR (
                              baked IS NULL AND stateid not in (1,8)
                              )
                          )
                      AND ARRAY [major_version, minor_version] = (
                          SELECT max(ARRAY[major_version,minor_version]) FROM
                              modules where m.uuid= uuid)

                GROUP BY print_style, recipe
                ),
                defaults AS (SELECT print_style, fileid AS recipe
                FROM default_print_style_recipes d
                WHERE not exists (SELECT 1
                                  FROM latest WHERE latest.recipe = d.fileid)
                )
                SELECT coalesce(ps.print_style, '(custom)') as print_style,
                       ps.title, coalesce(ps.recipe_type, 'web') as type,
                       ps.revised, ps.tag, ps.commit_id, la.count, la.bad
                FROM latest la LEFT JOIN print_style_recipes ps ON
                                    la.print_style = ps.print_style AND
                                    la.recipe = ps.fileid
                UNION ALL
                SELECT ps.print_style, ps.title, ps.recipe_type,
                       ps.revised, ps.tag, ps.commit_id, 0 AS count, 0 AS bad
                FROM defaults de JOIN print_style_recipes ps ON
                                    de.print_style = ps.print_style AND
                                    de.recipe = ps.fileid

            ORDER BY revised desc NULLS LAST, print_style

                """)
            for row in cursor.fetchall():
                styles.append({
                    'print_style': row['print_style'],
                    'title': row['title'],
                    'type': row['type'],
                    'revised': row['revised'],
                    'tag': row['tag'],
                    'commit_id': row['commit_id'],
                    'number': row['count'],
                    'bad': row['bad'],
                    'link': request.route_path('admin-print-style-single',
                                               style=row['print_style'])
                })
    return {'styles': styles}

Response: Returns a dictionary of all unique print_styles and their latest tag revision and recipe_type.


Instruction:
Create a Python 3 function to
return all books with any version of the given print style.

Response:
def admin_print_styles_single(request):
    """ Returns all books with any version of the given print style.

    Returns the print_style, recipe type, num books using the print_style,
    along with a dictionary of the book, author, revision date, recipe,
    tag of the print_style, and a link to the content.
    """
    style = request.matchdict['style']
    # do db search to get file id and other info on the print_style
    with db_connect(cursor_factory=DictCursor) as db_conn:
        with db_conn.cursor() as cursor:

            if style != '(custom)':
                cursor.execute("""
                    SELECT fileid, recipe_type, title
                    FROM default_print_style_recipes
                    WHERE print_style=%s
                    """, vars=(style,))
                info = cursor.fetchall()
                if len(info) < 1:
                    current_recipe = None
                    recipe_type = None
                    status = None

                else:
                    current_recipe = info[0]['fileid']
                    recipe_type = info[0]['recipe_type']
                    status = 'current'

                cursor.execute("""\
                    SELECT name, authors, lm.revised, lm.recipe, psr.tag,
                        f.sha1 as hash, psr.commit_id, uuid,
                        ident_hash(uuid, major_version, minor_version)
                    FROM modules as lm
                    LEFT JOIN print_style_recipes as psr
                    ON (psr.print_style = lm.print_style and
                        psr.fileid = lm.recipe)
                    LEFT JOIN files f ON psr.fileid = f.fileid
                    WHERE lm.print_style=%s
                    AND portal_type='Collection'
                    AND ARRAY [major_version, minor_version] = (
                        SELECT max(ARRAY[major_version,minor_version])
                        FROM modules WHERE lm.uuid = uuid)

                    ORDER BY psr.tag DESC;
                    """, vars=(style,))
            else:
                current_recipe = '(custom)'
                recipe_type = '(custom)'
                cursor.execute("""\
                    SELECT name, authors, lm.revised, lm.recipe, NULL as tag,
                        f.sha1 as hash, NULL as commit_id, uuid,
                        ident_hash(uuid, major_version, minor_version)
                    FROM modules as lm
                    JOIN files f ON lm.recipe = f.fileid
                    WHERE portal_type='Collection'
                    AND NOT EXISTS (
                        SELECT 1 from print_style_recipes psr
                        WHERE psr.fileid = lm.recipe)
                    AND ARRAY [major_version, minor_version] = (
                        SELECT max(ARRAY[major_version,minor_version])
                        FROM modules WHERE lm.uuid = uuid)
                    ORDER BY uuid, recipe, revised DESC;
                    """, vars=(style,))
                status = '(custom)'

            collections = []
            for row in cursor.fetchall():
                recipe = row['recipe']
                if (status != '(custom)' and
                        current_recipe is not None and
                        recipe != current_recipe):
                    status = 'stale'
                collections.append({
                    'title': row['name'].decode('utf-8'),
                    'authors': row['authors'],
                    'revised': row['revised'],
                    'recipe': row['hash'],
                    'recipe_link': request.route_path('get-resource',
                                                      hash=row['hash']),
                    'tag': row['tag'],
                    'ident_hash': row['ident_hash'],
                    'link': request.route_path('get-content',
                                               ident_hash=row['ident_hash']),
                    'status': status,
                    'status_link': request.route_path(
                        'admin-content-status-single', uuid=row['uuid']),

                })
    return {'number': len(collections),
            'collections': collections,
            'print_style': style,
            'recipe_type': recipe_type}


Instruction:
Write a Python 3 script to
return the list of API keys.

Response:
def get_api_keys(request):
    """Return the list of API keys."""
    with db_connect() as db_conn:
        with db_conn.cursor() as cursor:
            cursor.execute("""\
SELECT row_to_json(combined_rows) FROM (
  SELECT id, key, name, groups FROM api_keys
) AS combined_rows""")
            api_keys = [x[0] for x in cursor.fetchall()]

    return api_keys


Instruction:
Can you tell what is the following Python 3 function doing
def get_baking_statuses_sql(get_request):
    args = {}
    sort = get_request.get('sort', 'bpsa.created DESC')
    if (len(sort.split(" ")) != 2 or
            sort.split(" ")[0] not in SORTS_DICT.keys() or
            sort.split(" ")[1] not in ARROW_MATCH.keys()):
        raise httpexceptions.HTTPBadRequest(
            'invalid sort: {}'.format(sort))
    if sort == "STATE ASC" or sort == "STATE DESC":
        sort = 'bpsa.created DESC'
    uuid_filter = get_request.get('uuid', '').strip()
    author_filter = get_request.get('author', '').strip()
    latest_filter = get_request.get('latest', False)

    sql_filters = "WHERE"
    if latest_filter:
        sql_filters += """ ARRAY [m.major_version, m.minor_version] = (
         SELECT max(ARRAY[major_version,minor_version]) FROM
                   modules where m.uuid= uuid) AND """
    if uuid_filter != '':
        args['uuid'] = uuid_filter
        sql_filters += " m.uuid=%(uuid)s AND "
    if author_filter != '':
        author_filter = author_filter.decode('utf-8')
        sql_filters += " %(author)s=ANY(m.authors) "
        args["author"] = author_filter

    if sql_filters.endswith("AND "):
        sql_filters = sql_filters[:-4]
    if sql_filters == "WHERE":
        sql_filters = ""

    # FIXME  celery AsyncResult API is soooo sloow that this page takes
    # 2 min. or more to load on production.  As an workaround, this code
    # accesses the celery_taskmeta table directly. Need to remove that access
    # once we track enough state info ourselves. Want to track when queued,
    # started, ended, etc. for future monitoring of baking system performance
    # as well.
    # The 'limit 1' subselect is to ensure the "oldest identical version"
    # for recipes released as part of cnx-recipes (avoids one line per
    # identical recipe file in different releases, for a single baking job)

    statement = """
                       SELECT m.name, m.authors, m.uuid,
                       module_version(m.major_version,m.minor_version)
                          as current_version,
                       m.print_style,
                       CASE WHEN f.sha1 IS NOT NULL
                       THEN coalesce(dps.print_style,'(custom)')
                       ELSE dps.print_style
                       END AS recipe_name,
                       (select tag from print_style_recipes
                            where print_style = m.print_style
                                and fileid = m.recipe
                                order by revised asc limit 1) as recipe_tag,
                       coalesce(dps.fileid, m.recipe) as latest_recipe_id,
                       m.recipe as recipe_id,
                       f.sha1 as recipe,
                       m.module_ident,
                       ident_hash(m.uuid, m.major_version, m.minor_version),
                       bpsa.created, ctm.traceback,
                       CASE WHEN ctm.status = 'SUCCESS'
                           AND ms.statename = 'fallback'
                       THEN 'FALLBACK'
                       ELSE ctm.status
                       END as state
                FROM document_baking_result_associations AS bpsa
                INNER JOIN modules AS m USING (module_ident)
                INNER JOIN modulestates as ms USING (stateid)
                LEFT JOIN celery_taskmeta AS ctm
                    ON bpsa.result_id = ctm.task_id::uuid
                LEFT JOIN default_print_style_recipes as dps
                    ON dps.print_style = m.print_style
                LEFT JOIN latest_modules as lm
                    ON lm.uuid=m.uuid
                LEFT JOIN files f on m.recipe = f.fileid
                {}
                ORDER BY {};
                """.format(sql_filters, sort)
    args.update({'sort': sort})
    return statement, args

Response: Returns a SQL query that returns the list of baking statuses for the current project.


Instruction:
Here you have a function in Python 3, explain what it does
def admin_content_status(request):
    statement, sql_args = get_baking_statuses_sql(request.GET)

    states = []
    status_filters = request.params.getall('status_filter') or []
    state_icons = dict(STATE_ICONS)
    with db_connect(cursor_factory=DictCursor) as db_conn:
        with db_conn.cursor() as cursor:
            cursor.execute(statement, vars=sql_args)
            for row in cursor.fetchall():
                message = ''
                state = row['state'] or 'PENDING'
                if status_filters and state not in status_filters:
                    continue
                if state == 'FAILURE':  # pragma: no cover
                    if row['traceback'] is not None:
                        message = row['traceback'].split("\n")[-2]
                latest_recipe = row['latest_recipe_id']
                current_recipe = row['recipe_id']
                if (current_recipe is not None and
                        current_recipe != latest_recipe):
                    state += ' stale_recipe'
                state_icon = state
                if state[:7] == "SUCCESS" and len(state) > 7:
                    state_icon = 'unknown'
                states.append({
                    'title': row['name'].decode('utf-8'),
                    'authors': format_authors(row['authors']),
                    'uuid': row['uuid'],
                    'print_style': row['print_style'],
                    'print_style_link': request.route_path(
                        'admin-print-style-single', style=row['print_style']),
                    'recipe': row['recipe'],
                    'recipe_name': row['recipe_name'],
                    'recipe_tag': row['recipe_tag'],
                    'recipe_link': request.route_path(
                        'get-resource', hash=row['recipe']),
                    'created': row['created'],
                    'state': state,
                    'state_message': message,
                    'state_icon': state_icons.get(
                        state_icon, DEFAULT_ICON),
                    'status_link': request.route_path(
                        'admin-content-status-single', uuid=row['uuid']),
                    'content_link': request.route_path(
                        'get-content', ident_hash=row['ident_hash'])
                })
    sort = request.params.get('sort', 'bpsa.created DESC')
    sort_match = SORTS_DICT[sort.split(' ')[0]]
    sort_arrow = ARROW_MATCH[sort.split(' ')[1]]
    if sort == "STATE ASC":
        states.sort(key=lambda x: x['state'])
    if sort == "STATE DESC":
        states.sort(key=lambda x: x['state'], reverse=True)

    num_entries = request.params.get('number', 100) or 100
    page = request.params.get('page', 1) or 1
    try:
        page = int(page)
        num_entries = int(num_entries)
        start_entry = (page - 1) * num_entries
    except ValueError:
        raise httpexceptions.HTTPBadRequest(
            'invalid page({}) or entries per page({})'.
            format(page, num_entries))
    total_entries = len(states)
    states = states[start_entry: start_entry + num_entries]

    returns = sql_args
    returns.update({'start_entry': start_entry,
                    'num_entries': num_entries,
                    'page': page,
                    'total_entries': total_entries,
                    'states': states,
                    'sort_' + sort_match: sort_arrow,
                    'sort': sort,
                    'domain': request.host,
                    'latest_only': request.GET.get('latest', False),
                    'STATE_ICONS': STATE_ICONS,
                    'status_filters': status_filters or [
                        i[0] for i in STATE_ICONS]})
    return returns

Response: Returns a dictionary with the states and info of baking books and the filters from the GET request to pre - populate the form.


Instruction:
How would you explain what the following Python 3 function does
def admin_content_status_single(request):
    uuid = request.matchdict['uuid']
    try:
        UUID(uuid)
    except ValueError:
        raise httpexceptions.HTTPBadRequest(
            '{} is not a valid uuid'.format(uuid))

    statement, sql_args = get_baking_statuses_sql({'uuid': uuid})
    with db_connect(cursor_factory=DictCursor) as db_conn:
        with db_conn.cursor() as cursor:
            cursor.execute(statement, sql_args)
            modules = cursor.fetchall()
            if len(modules) == 0:
                raise httpexceptions.HTTPBadRequest(
                    '{} is not a book'.format(uuid))

            states = []
            collection_info = modules[0]

            for row in modules:
                message = ''
                state = row['state'] or 'PENDING'
                if state == 'FAILURE':  # pragma: no cover
                    if row['traceback'] is not None:
                        message = row['traceback']
                latest_recipe = row['latest_recipe_id']
                current_recipe = row['recipe_id']
                if (latest_recipe is not None and
                        current_recipe != latest_recipe):
                    state += ' stale_recipe'
                states.append({
                    'version': row['current_version'],
                    'recipe': row['recipe'],
                    'created': str(row['created']),
                    'state': state,
                    'state_message': message,
                })

    return {'uuid': str(collection_info['uuid']),
            'title': collection_info['name'].decode('utf-8'),
            'authors': format_authors(collection_info['authors']),
            'print_style': collection_info['print_style'],
            'current_recipe': collection_info['recipe_id'],
            'current_ident': collection_info['module_ident'],
            'current_state': states[0]['state'],
            'states': states}

Response: Returns a dictionary with all the past baking statuses of a single book.


Instruction:
Given the following Python 3 function, write the documentation
def admin_content_status_single_POST(request):
    args = admin_content_status_single(request)
    title = args['title']
    if args['current_state'] == 'SUCCESS':
        args['response'] = title + ' is not stale, no need to bake'
        return args

    with db_connect() as db_conn:
        with db_conn.cursor() as cursor:
            cursor.execute("SELECT stateid FROM modules WHERE module_ident=%s",
                           vars=(args['current_ident'],))
            data = cursor.fetchall()
            if len(data) == 0:
                raise httpexceptions.HTTPBadRequest(
                    'invalid module_ident: {}'.format(args['current_ident']))
            if data[0][0] == 5 or data[0][0] == 6:
                args['response'] = title + ' is already baking/set to bake'
                return args

            cursor.execute("""UPDATE modules SET stateid=5
                           WHERE module_ident=%s""",
                           vars=(args['current_ident'],))

            args['response'] = title + " set to bake!"

    return args

Response: Retetriggers baking for a given book.


Instruction:
How would you code a function in Python 3 to
insert the optional roles for the given module.

Response:
def _insert_optional_roles(cursor, model, ident):
    """Inserts the optional roles if values for the optional roles
    exist.
    """
    optional_roles = [
        # (<metadata-attr>, <db-role-id>,),
        ('translators', 4,),
        ('editors', 5,),
    ]
    for attr, role_id in optional_roles:
        roles = model.metadata.get(attr)
        if not roles:
            # Bail out, no roles for this type.
            continue
        usernames = [parse_user_uri(x['id']) for x in roles]
        cursor.execute("""\
INSERT INTO moduleoptionalroles (module_ident, roleid, personids)
VALUES (%s, %s, %s)""", (ident, role_id, usernames,))


Instruction:
Create a Python 3 function to
insert a module with the given metadata.

Response:
def _insert_metadata(cursor, model, publisher, message):
    """Insert a module with the given ``metadata``."""
    params = model.metadata.copy()
    params['publisher'] = publisher
    params['publication_message'] = message
    params['_portal_type'] = _model_to_portaltype(model)

    params['summary'] = str(cnxepub.DocumentSummaryFormatter(model))

    # Transform person structs to id lists for database array entry.
    for person_field in ATTRIBUTED_ROLE_KEYS:
        params[person_field] = [parse_user_uri(x['id'])
                                for x in params.get(person_field, [])]
    params['parent_ident_hash'] = parse_parent_ident_hash(model)

    # Assign the id and version if one is known.
    if model.ident_hash is not None:
        uuid, version = split_ident_hash(model.ident_hash,
                                         split_version=True)
        params['_uuid'] = uuid
        params['_major_version'], params['_minor_version'] = version
        # Lookup legacy ``moduleid``.
        cursor.execute("SELECT moduleid FROM latest_modules WHERE uuid = %s",
                       (uuid,))
        # There is the chance that a uuid and version have been set,
        #   but a previous publication does not exist. Therefore the
        #   moduleid will not be found. This happens on a pre-publication.
        try:
            moduleid = cursor.fetchone()[0]
        except TypeError:  # NoneType
            moduleid = None
        params['_moduleid'] = moduleid

        # Verify that uuid is reserved in document_contols. If not, add it.
        cursor.execute("SELECT * from document_controls where uuid = %s",
                       (uuid,))
        try:
            cursor.fetchone()[0]
        except TypeError:  # NoneType
            cursor.execute("INSERT INTO document_controls (uuid) VALUES (%s)",
                           (uuid,))

        created = model.metadata.get('created', None)
        # Format the statement to accept the identifiers.
        stmt = MODULE_INSERTION_TEMPLATE.format(**{
            '__uuid__': "%(_uuid)s::uuid",
            '__major_version__': "%(_major_version)s",
            '__minor_version__': "%(_minor_version)s",
            '__moduleid__': moduleid is None and "DEFAULT" or "%(_moduleid)s",
            '__created__': created is None and "DEFAULT" or "%(created)s",
        })
    else:
        created = model.metadata.get('created', None)
        # Format the statement for defaults.
        stmt = MODULE_INSERTION_TEMPLATE.format(**{
            '__uuid__': "DEFAULT",
            '__major_version__': "DEFAULT",
            '__minor_version__': "DEFAULT",
            '__moduleid__': "DEFAULT",
            '__created__': created is None and "DEFAULT" or "%(created)s",
        })

    # Insert the metadata
    cursor.execute(stmt, params)
    module_ident, ident_hash = cursor.fetchone()
    # Insert optional roles
    _insert_optional_roles(cursor, model, module_ident)

    return module_ident, ident_hash


Instruction:
Can you tell what is the following Python 3 function doing
def _get_file_sha1(file):
    bits = file.read()
    file.seek(0)
    h = hashlib.new('sha1', bits).hexdigest()
    return h

Response: Return the SHA1 hash of the given a file - like object as file.


Instruction:
Implement a function in Python 3 to
insert the file and media_type into the files table.

Response:
def _insert_file(cursor, file, media_type):
    """Upsert the ``file`` and ``media_type`` into the files table.
    Returns the ``fileid`` and ``sha1`` of the upserted file.

    """
    resource_hash = _get_file_sha1(file)
    cursor.execute("SELECT fileid FROM files WHERE sha1 = %s",
                   (resource_hash,))
    try:
        fileid = cursor.fetchone()[0]
    except (IndexError, TypeError):
        cursor.execute("INSERT INTO files (file, media_type) "
                       "VALUES (%s, %s)"
                       "RETURNING fileid",
                       (psycopg2.Binary(file.read()), media_type,))
        fileid = cursor.fetchone()[0]
    return fileid, resource_hash


Instruction:
Create a Python 3 function to
insert a resource into the modules_files table. This will create a new file entry and associates an existing one.

Response:
def _insert_resource_file(cursor, module_ident, resource):
    """Insert a resource into the modules_files table. This will
    create a new file entry or associates an existing one.
    """
    with resource.open() as file:
        fileid, _ = _insert_file(cursor, file, resource.media_type)

    # Is this file legitimately used twice within the same content?
    cursor.execute("""\
select
  (fileid = %s) as is_same_file
from module_files
where module_ident = %s and filename = %s""",
                   (fileid, module_ident, resource.filename,))
    try:
        is_same_file = cursor.fetchone()[0]
    except TypeError:  # NoneType
        is_same_file = None
    if is_same_file:
        # All is good, bail out.
        return
    elif is_same_file is not None:  # pragma: no cover
        # This means the file is not the same, but a filename
        #   conflict exists.
        # FFF At this time, it is impossible to get to this logic.
        raise Exception("filename conflict")

    args = (module_ident, fileid, resource.filename,)
    cursor.execute("""\
INSERT INTO module_files (module_ident, fileid, filename)
VALUES (%s, %s, %s)""", args)


Instruction:
How would you explain what the following Python 3 function does
def _insert_tree(cursor, tree, parent_id=None, index=0, is_collated=False):
    if isinstance(tree, dict):
        if tree['id'] == 'subcol':
            document_id = None
            title = tree['title']
        else:
            cursor.execute("""\
            SELECT module_ident, name
            FROM modules
            WHERE ident_hash(uuid,major_version,minor_version) = %s
            """, (tree['id'],))
            try:
                document_id, document_title = cursor.fetchone()
            except TypeError:  # NoneType
                raise ValueError("Missing published document for '{}'."
                                 .format(tree['id']))
            if tree.get('title', None):
                title = tree['title']
            else:
                title = document_title
        # TODO We haven't settled on a flag (name or value)
        #      to pin the node to a specific version.
        is_latest = True
        cursor.execute(TREE_NODE_INSERT,
                       dict(document_id=document_id, parent_id=parent_id,
                            title=title, child_order=index,
                            is_latest=is_latest, is_collated=is_collated))
        node_id = cursor.fetchone()[0]
        if 'contents' in tree:
            _insert_tree(cursor, tree['contents'], parent_id=node_id,
                         is_collated=is_collated)
    elif isinstance(tree, list):
        for tree_node in tree:
            _insert_tree(cursor, tree_node, parent_id=parent_id,
                         index=tree.index(tree_node), is_collated=is_collated)

Response: Inserts a binder tree into the archive.


Instruction:
Can you implement a function in Python 3 that
publishes the given model and returns its ident_hash.

Response:
def publish_model(cursor, model, publisher, message):
    """Publishes the ``model`` and return its ident_hash."""
    publishers = publisher
    if isinstance(publishers, list) and len(publishers) > 1:
        raise ValueError("Only one publisher is allowed. '{}' "
                         "were given: {}"
                         .format(len(publishers), publishers))
    module_ident, ident_hash = _insert_metadata(cursor, model,
                                                publisher, message)

    for resource in getattr(model, 'resources', []):
        _insert_resource_file(cursor, module_ident, resource)

    if isinstance(model, Document):
        html = bytes(cnxepub.DocumentContentFormatter(model))
        sha1 = hashlib.new('sha1', html).hexdigest()
        cursor.execute("SELECT fileid FROM files WHERE sha1 = %s", (sha1,))
        try:
            fileid = cursor.fetchone()[0]
        except TypeError:
            file_args = {
                'media_type': 'text/html',
                'data': psycopg2.Binary(html),
            }
            cursor.execute("""\
            insert into files (file, media_type)
            VALUES (%(data)s, %(media_type)s)
            returning fileid""", file_args)
            fileid = cursor.fetchone()[0]
        args = {
            'module_ident': module_ident,
            'filename': 'index.cnxml.html',
            'fileid': fileid,
        }
        cursor.execute("""\
        INSERT INTO module_files
          (module_ident, fileid, filename)
        VALUES
          (%(module_ident)s, %(fileid)s, %(filename)s)""", args)

    elif isinstance(model, Binder):
        tree = cnxepub.model_to_tree(model)
        tree = _insert_tree(cursor, tree)
    return ident_hash


Instruction:
Write a Python 3 function for
publishing the model and returns its ident_hash.

Response:
def publish_composite_model(cursor, model, parent_model, publisher, message):
    """Publishes the ``model`` and return its ident_hash."""
    if not (isinstance(model, CompositeDocument) or
            (isinstance(model, Binder) and
                model.metadata.get('type') == 'composite-chapter')):
        raise ValueError("This function only publishes Composite"
                         "objects. '{}' was given.".format(type(model)))
    if issequence(publisher) and len(publisher) > 1:
        raise ValueError("Only one publisher is allowed. '{}' "
                         "were given: {}"
                         .format(len(publisher), publisher))
    module_ident, ident_hash = _insert_metadata(cursor, model,
                                                publisher, message)

    model.id, model.metadata['version'] = split_ident_hash(ident_hash)
    model.set_uri('cnx-archive', ident_hash)

    for resource in model.resources:
        _insert_resource_file(cursor, module_ident, resource)

    if isinstance(model, CompositeDocument):
        html = bytes(cnxepub.DocumentContentFormatter(model))
        fileid, _ = _insert_file(cursor, io.BytesIO(html), 'text/html')
        file_arg = {
            'module_ident': module_ident,
            'parent_ident_hash': parent_model.ident_hash,
            'fileid': fileid,
        }
        cursor.execute("""\
        INSERT INTO collated_file_associations
          (context, item, fileid)
        VALUES
          ((SELECT module_ident FROM modules
            WHERE ident_hash(uuid, major_version, minor_version)
           = %(parent_ident_hash)s),
            %(module_ident)s, %(fileid)s)""", file_arg)

    return ident_hash


Instruction:
Write a Python 3 function for
publishing a given module s collated content in the context of the parent_model.

Response:
def publish_collated_document(cursor, model, parent_model):
    """Publish a given `module`'s collated content in the context of
    the `parent_model`. Note, the model's content is expected to already
    have the collated content. This will just persist that content to
    the archive.

    """
    html = bytes(cnxepub.DocumentContentFormatter(model))
    sha1 = hashlib.new('sha1', html).hexdigest()
    cursor.execute("SELECT fileid FROM files WHERE sha1 = %s", (sha1,))
    try:
        fileid = cursor.fetchone()[0]
    except TypeError:
        file_args = {
            'media_type': 'text/html',
            'data': psycopg2.Binary(html),
        }
        cursor.execute("""\
        INSERT INTO files (file, media_type)
        VALUES (%(data)s, %(media_type)s)
        RETURNING fileid""", file_args)
        fileid = cursor.fetchone()[0]
    args = {
        'module_ident_hash': model.ident_hash,
        'parent_ident_hash': parent_model.ident_hash,
        'fileid': fileid,
    }
    stmt = """\
INSERT INTO collated_file_associations (context, item, fileid)
VALUES
  ((SELECT module_ident FROM modules
    WHERE ident_hash(uuid, major_version, minor_version)
   = %(parent_ident_hash)s),
   (SELECT module_ident FROM modules
    WHERE ident_hash(uuid, major_version, minor_version)
   = %(module_ident_hash)s),
   %(fileid)s)"""
    cursor.execute(stmt, args)


Instruction:
Implement a Python 3 function for
publishing a given collated tree.

Response:
def publish_collated_tree(cursor, tree):
    """Publish a given collated `tree` (containing newly added
    `CompositeDocument` objects and number inforation)
    alongside the original tree.

    """
    tree = _insert_tree(cursor, tree, is_collated=True)
    return tree


Instruction:
Write a Python 3 script to
republish the Binders that share Documents in the publication context.

Response:
def republish_binders(cursor, models):
    """Republish the Binders that share Documents in the publication context.
    This needs to be given all the models in the publication context."""
    documents = set([])
    binders = set([])
    history_mapping = {}  # <previous-ident-hash>: <current-ident-hash>
    if not isinstance(models, (list, tuple, set,)):
        raise TypeError("``models`` Must be a sequence of model objects."
                        "We were given: {}".format(models))
    for model in models:
        if isinstance(model, (cnxepub.Binder,)):
            binders.add(split_ident_hash(model.ident_hash)[0])
            for doc in cnxepub.flatten_to_documents(model):
                documents.add(split_ident_hash(doc.ident_hash))
        else:
            documents.add(split_ident_hash(model.ident_hash))

    to_be_republished = []
    # What binders are these documents a part of?
    for (uuid, version) in documents:
        ident_hash = join_ident_hash(uuid, version)
        previous_ident_hash = get_previous_publication(cursor, ident_hash)
        if previous_ident_hash is None:
            # Has no prior existence.
            continue
        else:
            history_mapping[previous_ident_hash] = ident_hash
        cursor.execute("""\
WITH RECURSIVE t(nodeid, parent_id, documentid, path) AS (
  SELECT tr.nodeid, tr.parent_id, tr.documentid, ARRAY[tr.nodeid]
  FROM trees tr
  WHERE tr.documentid = (
    SELECT module_ident FROM modules
    WHERE ident_hash(uuid, major_version, minor_version) = %s)
UNION ALL
  SELECT c.nodeid, c.parent_id, c.documentid, path || ARRAY[c.nodeid]
  FROM trees c JOIN t ON (c.nodeid = t.parent_id)
  WHERE not c.nodeid = ANY(t.path)
)
SELECT ident_hash(uuid, major_version, minor_version)
FROM t JOIN latest_modules m ON (t.documentid = m.module_ident)
WHERE t.parent_id IS NULL
""",
                       (previous_ident_hash,))
        to_be_republished.extend([split_ident_hash(x[0])
                                  for x in cursor.fetchall()])
    to_be_republished = set(to_be_republished)

    republished_ident_hashes = []
    # Republish the Collections set.
    for (uuid, version) in to_be_republished:
        if uuid in binders:
            # This binder is already in the publication context,
            # don't try to publish it again.
            continue
        ident_hash = join_ident_hash(uuid, version)
        bumped_version = bump_version(cursor, uuid, is_minor_bump=True)
        republished_ident_hash = republish_collection(cursor, ident_hash,
                                                      version=bumped_version)
        # Set the identifier history.
        history_mapping[ident_hash] = republished_ident_hash
        rebuild_collection_tree(cursor, ident_hash, history_mapping)
        republished_ident_hashes.append(republished_ident_hash)

    return republished_ident_hashes


Instruction:
Explain what the following Python 3 code does
def get_previous_publication(cursor, ident_hash):
    cursor.execute("""\
WITH contextual_module AS (
  SELECT uuid, module_ident
  FROM modules
  WHERE ident_hash(uuid, major_version, minor_version) = %s)
SELECT ident_hash(m.uuid, m.major_version, m.minor_version)
FROM modules AS m JOIN contextual_module AS context ON (m.uuid = context.uuid)
WHERE
  m.module_ident < context.module_ident
ORDER BY revised DESC
LIMIT 1""", (ident_hash,))
    try:
        previous_ident_hash = cursor.fetchone()[0]
    except TypeError:  # NoneType
        previous_ident_hash = None
    return previous_ident_hash

Response: Get the previous publication of the given ident - hash.


Instruction:
Write a Python 3 function that can
bump to the next available version of the given content identified by uuid.

Response:
def bump_version(cursor, uuid, is_minor_bump=False):
    """Bump to the next version of the given content identified
    by ``uuid``. Returns the next available version as a version tuple,
    containing major and minor version.
    If ``is_minor_bump`` is ``True`` the version will minor bump. That is
    1.2 becomes 1.3 in the case of Collections. And 2 becomes 3 for
    Modules regardless of this option.
    """
    cursor.execute("""\
SELECT portal_type, major_version, minor_version
FROM latest_modules
WHERE uuid = %s::uuid""", (uuid,))
    type_, major_version, minor_version = cursor.fetchone()
    incr = 1
    if type_ == 'Collection' and is_minor_bump:
        minor_version = minor_version + incr
    else:
        major_version = major_version + incr
    return (major_version, minor_version,)


Instruction:
Make a summary of the following Python 3 code
def republish_collection(cursor, ident_hash, version):
    if not isinstance(version, (list, tuple,)):
        split_version = version.split('.')
        if len(split_version) == 1:
            split_version.append(None)
        version = tuple(split_version)
    major_version, minor_version = version

    cursor.execute("""\
WITH previous AS (
  SELECT module_ident
  FROM modules
  WHERE ident_hash(uuid, major_version, minor_version) = %s),
inserted AS (
  INSERT INTO modules
    (uuid, major_version, minor_version, revised,
     portal_type, moduleid,
     name, created, language,
     submitter, submitlog,
     abstractid, licenseid, parent, parentauthors,
     authors, maintainers, licensors,
     google_analytics, buylink,
     stateid, doctype)
  SELECT
    uuid, %s, %s, CURRENT_TIMESTAMP,
    portal_type, moduleid,
    name, created, language,
    submitter, submitlog,
    abstractid, licenseid, parent, parentauthors,
    authors, maintainers, licensors,
    google_analytics, buylink,
    stateid, doctype
  FROM modules AS m JOIN previous AS p ON (m.module_ident = p.module_ident)
  RETURNING
    ident_hash(uuid, major_version, minor_version) AS ident_hash,
    module_ident),
keywords AS (
  INSERT INTO modulekeywords (module_ident, keywordid)
  SELECT i.module_ident, keywordid
  FROM modulekeywords AS mk, inserted AS i, previous AS p
  WHERE mk.module_ident = p.module_ident),
tags AS (
  INSERT INTO moduletags (module_ident, tagid)
  SELECT i.module_ident, tagid
  FROM moduletags AS mt, inserted AS i, previous AS p
  WHERE mt.module_ident = p.module_ident)
SELECT ident_hash FROM inserted""",
                   (ident_hash, major_version, minor_version,))
    repub_ident_hash = cursor.fetchone()[0]
    return repub_ident_hash

Response: Republish the collection identified as ident_hash with the given version.


Instruction:
Make a summary of the following Python 3 code
def rebuild_collection_tree(cursor, ident_hash, history_map):
    collection_tree_sql = """\
WITH RECURSIVE t(nodeid, parent_id, documentid, title, childorder, latest,
                 ident_hash, path) AS (
  SELECT
    tr.nodeid, tr.parent_id, tr.documentid,
    tr.title, tr.childorder, tr.latest,
    (SELECT ident_hash(uuid, major_version, minor_version)
     FROM modules
     WHERE module_ident = tr.documentid) AS ident_hash,
    ARRAY[tr.nodeid]
  FROM trees AS tr
  WHERE tr.documentid = (
    SELECT module_ident
    FROM modules
    WHERE ident_hash(uuid, major_version, minor_version) = %s)
    AND tr.is_collated = FALSE
UNION ALL
  SELECT
    c.nodeid, c.parent_id, c.documentid, c.title, c.childorder, c.latest,
    (SELECT ident_hash(uuid, major_version, minor_version)
     FROM modules
     WHERE module_ident = c.documentid) AS ident_hash,
    path || ARRAY[c.nodeid]
  FROM trees AS c JOIN t ON (c.parent_id = t.nodeid)
  WHERE not c.nodeid = ANY(t.path) AND c.is_collated = FALSE
)
SELECT row_to_json(row) FROM (SELECT * FROM t) AS row"""

    tree_insert_sql = """\
INSERT INTO trees
  (nodeid, parent_id,
   documentid,
   title, childorder, latest)
VALUES
  (DEFAULT, %(parent_id)s,
   (SELECT module_ident
    FROM modules
    WHERE ident_hash(uuid, major_version, minor_version) = \
          %(ident_hash)s),
   %(title)s, %(childorder)s, %(latest)s)
RETURNING nodeid"""

    def get_tree():
        cursor.execute(collection_tree_sql, (ident_hash,))
        for row in cursor.fetchall():
            yield row[0]

    def insert(fields):
        cursor.execute(tree_insert_sql, fields)
        results = cursor.fetchone()[0]
        return results

    tree = {}  # {<current-nodeid>: {<row-data>...}, ...}
    children = {}  # {<nodeid>: [<child-nodeid>, ...], <child-nodeid>: [...]}
    for node in get_tree():
        tree[node['nodeid']] = node
        children.setdefault(node['parent_id'], [])
        children[node['parent_id']].append(node['nodeid'])

    def build_tree(nodeid, parent_id):
        data = tree[nodeid]
        data['parent_id'] = parent_id
        if history_map.get(data['ident_hash']) is not None \
           and (data['latest'] or parent_id is None):
            data['ident_hash'] = history_map[data['ident_hash']]
        new_nodeid = insert(data)
        for child_nodeid in children.get(nodeid, []):
            build_tree(child_nodeid, new_nodeid)

    root_node = children[None][0]
    build_tree(root_node, None)

Response: Rebuild the tree based on the old tree and the new tree based on the new tree


Instruction:
Can you create a Python 3 function that
publishes a new resource.

Response:
def publish(request):
    """Accept a publication request at form value 'epub'"""
    if 'epub' not in request.POST:
        raise httpexceptions.HTTPBadRequest("Missing EPUB in POST body.")

    is_pre_publication = asbool(request.POST.get('pre-publication'))
    epub_upload = request.POST['epub'].file
    try:
        epub = cnxepub.EPUB.from_file(epub_upload)
    except:  # noqa: E722
        raise httpexceptions.HTTPBadRequest('Format not recognized.')

    # Make a publication entry in the database for status checking
    # the publication. This also creates publication entries for all
    # of the content in the EPUB.
    with db_connect() as db_conn:
        with db_conn.cursor() as cursor:
            epub_upload.seek(0)
            publication_id, publications = add_publication(
                cursor, epub, epub_upload, is_pre_publication)

    # Poke at the publication & lookup its state.
    state, messages = poke_publication_state(publication_id)

    response_data = {
        'publication': publication_id,
        'mapping': publications,
        'state': state,
        'messages': messages,
    }
    return response_data


Instruction:
Given the following Python 3 function, write the documentation
def get_accept_license(request):
    publication_id = request.matchdict['id']
    user_id = request.matchdict['uid']

    # FIXME Is this an active publication?
    # TODO Verify the accepting user is the one making the request.

    # For each pending document, accept the license.
    with db_connect() as db_conn:
        with db_conn.cursor() as cursor:
            cursor.execute("""
SELECT row_to_json(combined_rows) FROM (
SELECT
  pd.uuid AS id,
  ident_hash(pd.uuid, pd.major_version, pd.minor_version) \
    AS ident_hash,
  accepted AS is_accepted
FROM
  pending_documents AS pd
  NATURAL JOIN license_acceptances AS la
WHERE pd.publication_id = %s AND user_id = %s
) as combined_rows;""",
                           (publication_id, user_id))
            user_documents = [r[0] for r in cursor.fetchall()]

    return {'publication_id': publication_id,
            'user_id': user_id,
            'documents': user_documents,
            }

Response: This returns the JSON data for a user to view the license(s) that have been accepted for a publication.


Instruction:
Implement a Python 3 function for
allowing the user to accept the license for a specific publication.

Response:
def post_accept_license(request):
    """Allows the user (at ``uid``) to accept the license(s) for
    a publication (at ``id``).
    """
    publication_id = request.matchdict['id']
    uid = request.matchdict['uid']

    # TODO Verify the accepting user is the one making the request.
    #      They could be authenticated but not be the license acceptor.

    post_data = request.json
    accepted = []
    denied = []
    try:
        documents = post_data['documents']
        for doc_acceptance in documents:
            if doc_acceptance['is_accepted'] is None:
                continue
            elif doc_acceptance['is_accepted']:
                accepted.append(doc_acceptance['id'])
            else:
                denied.append(doc_acceptance['id'])
    except KeyError:
        raise httpexceptions.BadRequest("Posted data is invalid.")

    # For each pending document, accept/deny the license.
    with db_connect() as db_conn:
        with db_conn.cursor() as cursor:
            accept_publication_license(cursor, publication_id, uid,
                                       accepted, True)
            accept_publication_license(cursor, publication_id, uid,
                                       denied, False)

    location = request.route_url('publication-license-acceptance',
                                 id=publication_id, uid=uid)
    # Poke publication to change state.
    poke_publication_state(publication_id)
    return httpexceptions.HTTPFound(location=location)


Instruction:
Implement a Python 3 function for
invoking the baking process - trigger post - publication

Response:
def bake_content(request):
    """Invoke the baking process - trigger post-publication"""
    ident_hash = request.matchdict['ident_hash']
    try:
        id, version = split_ident_hash(ident_hash)
    except IdentHashError:
        raise httpexceptions.HTTPNotFound()

    if not version:
        raise httpexceptions.HTTPBadRequest('must specify the version')

    with db_connect() as db_conn:
        with db_conn.cursor() as cursor:
            cursor.execute("""\
SELECT bool(portal_type = 'Collection'), stateid, module_ident
FROM modules
WHERE ident_hash(uuid, major_version, minor_version) = %s
""", (ident_hash,))
            try:
                is_binder, stateid, module_ident = cursor.fetchone()
            except TypeError:
                raise httpexceptions.HTTPNotFound()
            if not is_binder:
                raise httpexceptions.HTTPBadRequest(
                    '{} is not a book'.format(ident_hash))

            if stateid == 5:
                cursor.execute("""\
SELECT pg_notify('post_publication',
'{"module_ident": '||%s||',
  "ident_hash": "'||%s||'",
  "timestamp": "'||CURRENT_TIMESTAMP||'"}')
""", (module_ident, ident_hash))
            else:
                cursor.execute("""\
UPDATE modules SET stateid = 5
WHERE ident_hash(uuid, major_version, minor_version) = %s
""", (ident_hash,))


Instruction:
Can you generate the documentation for the following Python 3 function
def includeme(config):
    global cache_manager
    settings = config.registry.settings
    cache_manager = CacheManager(**parse_cache_config_options(settings))

Response: Configures the caching manager


Instruction:
Can you create a Python 3 function that
creates a Postgres Notification Event object from a Notify object.

Response:
def create_pg_notify_event(notif):
    """A factory for creating a Postgres Notification Event
    (an object inheriting from `cnxpublishing.events.PGNotifyEvent`)
    given `notif`, a `psycopg2.extensions.Notify` object.

    """
    # TODO Lookup registered events via getAllUtilitiesRegisteredFor
    #      for class mapping.
    if notif.channel not in _CHANNEL_MAPPER:
        cls = _CHANNEL_MAPPER[None]
    else:
        cls = _CHANNEL_MAPPER[notif.channel]
    return cls(notif)


Instruction:
Explain what the following Python 3 code does
def map(self, from_obj, to_type, ignore_case=False, allow_none=False, excluded=None):
        if (from_obj is None) and allow_none:
            return None
        else:
            # one of the tests is explicitly checking for an attribute error on __dict__ if it's not set
            from_obj.__dict__

        inst = to_type()
        key_from = from_obj.__class__.__name__
        key_to = to_type.__name__

        def not_private(s):
            return not s.startswith('_')

        def not_excluded(s):
            return not (excluded and s in excluded)

        from_obj_attributes = getmembers(from_obj, lambda a: not isroutine(a))
        from_obj_dict = {k: v for k, v in from_obj_attributes
                         if not_private(k) and not_excluded(k)}

        to_obj_attributes = getmembers(inst, lambda a: not isroutine(a))
        to_obj_dict = {k: v for k, v in to_obj_attributes if not_private(k)}

        if ignore_case:
            from_props = CaseDict(from_obj_dict)
            to_props = CaseDict(to_obj_dict)
        else:
            from_props = from_obj_dict
            to_props = to_obj_dict

        for prop in to_props:
            if self.mappings is not None \
                    and key_from in self.mappings \
                    and key_to in self.mappings[key_from]:
                if prop in self.mappings[key_from][key_to]:
                    # take mapping function
                    try:
                        fnc = self.mappings[key_from][key_to][prop]
                        if fnc is not None:
                            setattr(inst, prop, fnc(from_obj))
                            # none suppress mapping
                    except Exception:
                        raise ObjectMapperException("Invalid mapping function while setting property {0}.{1}".
                                                    format(inst.__class__.__name__, prop))

                else:
                    # try find property with the same name in the source
                    if prop in from_props:
                        setattr(inst, prop, from_props[prop])
                        # case when target attribute is not mapped (can be extended)
            else:
                raise ObjectMapperException("No mapping defined for {0} -> {1}".format(key_from, key_to))

        return inst

Response: Method for creating a new object from one object to another.


Instruction:
Here you have a function in Python 3, explain what it does
def get(self, key, default=_sentinel):
        tup = self._data.get(key.lower())
        if tup is not None:
            return tup[1]
        elif default is not _sentinel:
            return default
        else:
            return None

Response: Gets the value of the key from the key.


Instruction:
Can you tell what is the following Python 3 function doing
def pop(self, key, default=_sentinel):
        if default is not _sentinel:
            tup = self._data.pop(key.lower(), default)
        else:
            tup = self._data.pop(key.lower())
        if tup is not default:
            return tup[1]
        else:
            return default

Response: Removes the specified key and returns the corresponding value. If default is not given KeyError is raised.


Instruction:
Here you have a function in Python 3, explain what it does
def reversals(series, left=False, right=False):
    series = iter(series)

    x_last, x = next(series), next(series)
    d_last = (x - x_last)

    if left:
        yield x_last
    for x_next in series:
        if x_next == x:
            continue
        d_next = x_next - x
        if d_last * d_next < 0:
            yield x
        x_last, x = x, x_next
        d_last = d_next
    if right:
        yield x_next

Response: Iterate over the series of reversal points.


Instruction:
Explain what the following Python 3 code does
def extract_cycles(series, left=False, right=False):
    points = deque()

    for x in reversals(series, left=left, right=right):
        points.append(x)
        while len(points) >= 3:
            # Form ranges X and Y from the three most recent points
            X = abs(points[-2] - points[-1])
            Y = abs(points[-3] - points[-2])

            if X < Y:
                # Read the next point
                break
            elif len(points) == 3:
                # Y contains the starting point
                # Count Y as one-half cycle and discard the first point
                yield points[0], points[1], 0.5
                points.popleft()
            else:
                # Count Y as one cycle and discard the peak and the valley of Y
                yield points[-3], points[-2], 1.0
                last = points.pop()
                points.pop()
                points.pop()
                points.append(last)
    else:
        # Count the remaining ranges as one-half cycles
        while len(points) > 1:
            yield points[0], points[1], 0.5
            points.popleft()

Response: Iterate cycles in the series.


Instruction:
Can you write a function in Python 3 where it
counts the number of cycles in a series.

Response:
def count_cycles(series, ndigits=None, left=False, right=False):
    """Count cycles in the series.

    Parameters
    ----------
    series : iterable sequence of numbers
    ndigits : int, optional
        Round cycle magnitudes to the given number of digits before counting.
    left: bool, optional
        If True, treat the first point in the series as a reversal.
    right: bool, optional
        If True, treat the last point in the series as a reversal.

    Returns
    -------
    A sorted list containing pairs of cycle magnitude and count.
    One-half cycles are counted as 0.5, so the returned counts may not be
    whole numbers.
    """
    counts = defaultdict(float)
    round_ = _get_round_function(ndigits)

    for low, high, mult in extract_cycles(series, left=left, right=right):
        delta = round_(abs(high - low))
        counts[delta] += mult
    return sorted(counts.items())


Instruction:
Can you create a Python 3 function that
returns a FST that renders a given node.

Response:
def render(node, strict=False):
    """Recipe to render a given FST node.

    The FST is composed of branch nodes which are either lists or dicts
    and of leaf nodes which are strings. Branch nodes can have other
    list, dict or leaf nodes as childs.

    To render a string, simply output it. To render a list, render each
    of its elements in order. To render a dict, you must follow the
    node's entry in the nodes_rendering_order dictionary and its
    dependents constraints.

    This function hides all this algorithmic complexity by returning
    a structured rendering recipe, whatever the type of node. But even
    better, you should subclass the RenderWalker which simplifies
    drastically working with the rendered FST.

    The recipe is a list of steps, each step correspond to a child and is actually a 3-uple composed of the following fields:

    - `key_type` is a string determining the type of the child in the second field (`item`) of the tuple. It can be one of:

      - 'constant': the child is a string
      - 'node': the child is a dict
      - 'key': the child is an element of a dict
      - 'list': the child is a list
      - 'formatting': the child is a list specialized in formatting

    - `item` is the child itself: either a string, a dict or a list.
    - `render_key` gives the key used to access this child from the parent node. It's a string if the node is a dict or a number if its a list.

    Please note that "bool" `key_types` are never rendered, that's why
    they are not shown here.
    """
    if isinstance(node, list):
        return render_list(node)

    elif isinstance(node, dict):
        return render_node(node, strict=strict)

    else:
        raise NotImplementedError("You tried to render a %s. Only list and dicts can be rendered." % node.__class__.__name__)


Instruction:
How would you implement a function in Python 3 that
returns the FST node located at the given path

Response:
def path_to_node(tree, path):
    """FST node located at the given path"""
    if path is None:
        return None

    node = tree

    for key in path:
        node = child_by_key(node, key)

    return node


Instruction:
Given the following Python 3 function, write the documentation
def before_constant(self, constant, key):
        newlines_split = split_on_newlines(constant)

        for c in newlines_split:
            if is_newline(c):
                self.current.advance_line()
                # if target line is passed
                if self.current.line > self.target.line:
                    return self.STOP

            else:
                advance_by = len(c)
                if self.is_on_targetted_node(advance_by):
                    self.found_path = deepcopy(self.current_path)
                    return self.STOP
                self.current.advance_columns(advance_by)

Response: Determine if we re on the targetted node.


Instruction:
Create a Python 3 function to
return the prefix for a given multicodec name.

Response:
def get_prefix(multicodec):
    """
    Returns prefix for a given multicodec

    :param str multicodec: multicodec codec name
    :return: the prefix for the given multicodec
    :rtype: byte
    :raises ValueError: if an invalid multicodec name is provided
    """
    try:
        prefix = varint.encode(NAME_TABLE[multicodec])
    except KeyError:
        raise ValueError('{} multicodec is not supported.'.format(multicodec))
    return prefix


Instruction:
Can you tell what is the following Python 3 function doing
def add_prefix(multicodec, bytes_):
    prefix = get_prefix(multicodec)
    return b''.join([prefix, bytes_])

Response: Adds multicodec prefix to the given bytes input


Instruction:
Implement a function in Python 3 to
remove prefix from a prefixed data bytes

Response:
def remove_prefix(bytes_):
    """
    Removes prefix from a prefixed data

    :param bytes bytes_: multicodec prefixed data bytes
    :return: prefix removed data bytes
    :rtype: bytes
    """
    prefix_int = extract_prefix(bytes_)
    prefix = varint.encode(prefix_int)
    return bytes_[len(prefix):]


Instruction:
Can you generate a brief explanation for the following Python 3 code
def get_codec(bytes_):
    prefix = extract_prefix(bytes_)
    try:
        return CODE_TABLE[prefix]
    except KeyError:
        raise ValueError('Prefix {} not present in the lookup table'.format(prefix))

Response: Gets the codec used for prefix the multicodec prefixed data


Instruction:
How would you explain what the following Python 3 function does
def capture(
    target_url,
    user_agent="archiveis (https://github.com/pastpages/archiveis)",
    proxies={}
):
    """
    Archives the provided URL using archive.is

    Returns the URL where the capture is stored.
    """
    # Put together the URL that will save our request
    domain = "http://archive.vn"
    save_url = urljoin(domain, "/submit/")

    # Configure the request headers
    headers = {
        'User-Agent': user_agent,
        "host": "archive.vn",
    }

    # Request a unique identifier for our activity
    logger.debug("Requesting {}".format(domain + "/"))
    get_kwargs = dict(
        timeout=120,
        allow_redirects=True,
        headers=headers,
    )
    if proxies:
        get_kwargs['proxies'] = proxies
    response = requests.get(domain + "/", **get_kwargs)
    response.raise_for_status()

    # It will need to be parsed from the homepage response headers
    html = str(response.content)
    try:
        unique_id = html.split('name="submitid', 1)[1].split('value="', 1)[1].split('"', 1)[0]
        logger.debug("Unique identifier: {}".format(unique_id))
    except IndexError:
        logger.warn("Unable to extract unique identifier from archive.is. Submitting without it.")
        unique_id = None

    # Send the capture request to archive.is with the unique id included
    data = {
        "url": target_url,
        "anyway": 1,
    }
    if unique_id:
        data.update({"submitid": unique_id})

    post_kwargs = dict(
        timeout=120,
        allow_redirects=True,
        headers=headers,
        data=data
    )
    if proxies:
        post_kwargs['proxies'] = proxies

    logger.debug("Requesting {}".format(save_url))
    response = requests.post(save_url, **post_kwargs)
    response.raise_for_status()

    # There are a couple ways the header can come back
    if 'Refresh' in response.headers:
        memento = str(response.headers['Refresh']).split(';url=')[1]
        logger.debug("Memento from Refresh header: {}".format(memento))
        return memento
    if 'Location' in response.headers:
        memento = response.headers['Location']
        logger.debug("Memento from Location header: {}".format(memento))
        return memento
    logger.debug("Memento not found in response headers. Inspecting history.")
    for i, r in enumerate(response.history):
        logger.debug("Inspecting history request #{}".format(i))
        logger.debug(r.headers)
        if 'Location' in r.headers:
            memento = r.headers['Location']
            logger.debug("Memento from the Location header of {} history response: {}".format(i+1, memento))
            return memento
    # If there's nothing at this point, throw an error
    logger.error("No memento returned by archive.is")
    logger.error("Status code: {}".format(response.status_code))
    logger.error(response.headers)
    logger.error(response.text)
    raise Exception("No memento returned by archive.is")

Response: Create a new object of the correct type for the given target_url.


Instruction:
Create a Python 3 function for
archiving the provided URL using archive.is.

Response:
def cli(url, user_agent):
    """
    Archives the provided URL using archive.is.
    """
    kwargs = {}
    if user_agent:
        kwargs['user_agent'] = user_agent
    archive_url = capture(url, **kwargs)
    click.echo(archive_url)


Instruction:
Write a Python 3 function for
getting the logo for a channel.

Response:
def get_channel_image(self, channel, img_size=300, skip_cache=False):
        """Get the logo for a channel"""
        from bs4 import BeautifulSoup
        from wikipedia.exceptions import PageError
        import re
        import wikipedia
        wikipedia.set_lang('fr')

        if not channel:
            _LOGGER.error('Channel is not set. Could not retrieve image.')
            return

        # Check if the image is in cache
        if channel in self._cache_channel_img and not skip_cache:
            img = self._cache_channel_img[channel]
            _LOGGER.debug('Cache hit: %s -> %s', channel, img)
            return img

        channel_info = self.get_channel_info(channel)
        query = channel_info['wiki_page']
        if not query:
            _LOGGER.debug('Wiki page is not set for channel %s', channel)
            return
        _LOGGER.debug('Query: %s', query)
        # If there is a max image size defined use it.
        if 'max_img_size' in channel_info:
            if img_size > channel_info['max_img_size']:
                _LOGGER.info(
                    'Requested image size is bigger than the max, '
                    'setting it to %s', channel_info['max_img_size']
                )
                img_size = channel_info['max_img_size']
        try:
            page = wikipedia.page(query)
            _LOGGER.debug('Wikipedia article title: %s', page.title)
            soup = BeautifulSoup(page.html(), 'html.parser')
            images = soup.find_all('img')
            img_src = None
            for i in images:
                if i['alt'].startswith('Image illustrative'):
                    img_src = re.sub(r'\d+px', '{}px'.format(img_size),
                                     i['src'])
            img = 'https:{}'.format(img_src) if img_src else None
            # Cache result
            self._cache_channel_img[channel] = img
            return img
        except PageError:
            _LOGGER.error('Could not fetch channel image for %s', channel)


Instruction:
Create a Python 3 function to
press a key from the cache.

Response:
def press_key(self, key, mode=0):
        '''
        modes:
            0 -> simple press
            1 -> long press
            2 -> release after long press
        '''
        if isinstance(key, str):
            assert key in KEYS, 'No such key: {}'.format(key)
            key = KEYS[key]
        _LOGGER.info('Press key %s', self.__get_key_name(key))
        return self.rq('01', OrderedDict([('key', key), ('mode', mode)]))


Instruction:
Can you generate the documentation for the following Python 3 function
def forwards(self, orm):
        "Write your forwards methods here."
        # Note: Remember to use orm['appname.ModelName'] rather than "from appname.models..."
        for translation in orm['people.PersonTranslation'].objects.all():
            if translation.language in ['en', 'de']:
                translation.roman_first_name = translation.first_name
                translation.roman_last_name = translation.last_name
            else:
                translation.non_roman_first_name = translation.first_name
                translation.non_roman_last_name = translation.last_name
            translation.save()

Response: Write your forwards methods here.


Instruction:
Write a Python 3 function that can
write your forwards methods here.

Response:
def forwards(self, orm):
        "Write your forwards methods here."
        for translation in orm['people.PersonTranslation'].objects.all():
            translation.person.roman_first_name = translation.roman_first_name
            translation.person.roman_last_name = translation.roman_last_name
            translation.person.non_roman_first_name = translation.non_roman_first_name
            translation.person.non_roman_last_name = translation.non_roman_last_name
            translation.person.save()