Instruction:
Can you tell what is the following Python 3 function doing
def nearest(self, ver):
        if not isinstance(ver, Version):
            ver = Version(ver)

        if ver in OFFICIAL_VERSIONS:
            return ver

        # We might not have an exact match for that.
        # See if we have one that's newer than the grid we're looking at.
        versions = list(OFFICIAL_VERSIONS)
        versions.sort(reverse=True)
        best = None
        for candidate in versions:
            # Due to ambiguities, we might have an exact match and not know it.
            # '2.0' will not hash to the same value as '2.0.0', but both are
            # equivalent.
            if candidate == ver:
                # We can't beat this, make a note of the match for later
                return candidate

            # If we have not seen a better candidate, and this is older
            # then we may have to settle for that.
            if (best is None) and (candidate < ver):
                warnings.warn('This version of hszinc does not yet '\
                            'support version %s, please seek a newer version '\
                            'or file a bug.  Closest (older) version supported is %s.'\
                            % (ver, candidate))
                return candidate

            # Probably the best so far, but see if we can go closer
            if candidate > ver:
                best = candidate

        # Unhappy path, no best option?  This should not happen.
        assert best is not None
        warnings.warn('This version of hszinc does not yet '\
                    'support version %s, please seek a newer version '\
                    'or file a bug.  Closest (newer) version supported is %s.'\
                    % (ver, best))
        return best

Response: Retrieve the official version nearest the one given.


Instruction:
Write a Python 3 script to
encrypt a file with gpg and random generated password

Response:
def encrypt_files(selected_host, only_link, file_name):
    """
    Encrypts file with gpg and random generated password
    """
    if ENCRYPTION_DISABLED:
        print('For encryption please install gpg')
        exit()
    passphrase = '%030x' % random.randrange(16**30)
    source_filename = file_name
    cmd = 'gpg --batch --symmetric --cipher-algo AES256 --passphrase-fd 0 ' \
          '--output - {}'.format(source_filename)
    encrypted_output = Popen(shlex.split(cmd), stdout=PIPE, stdin=PIPE, stderr=PIPE)
    encrypted_data = encrypted_output.communicate(passphrase.encode())[0]
    return upload_files(encrypted_data, selected_host, only_link, file_name)+'#'+passphrase


Instruction:
Write a Python 3 function for
checking if the file size is greater than max_size

Response:
def check_max_filesize(chosen_file, max_size):
    """
    Checks file sizes for host
    """
    if os.path.getsize(chosen_file) > max_size:
        return False
    else:
        return True


Instruction:
Create a Python 3 function to
take the arguments and returns a string that can be used to create a new resource.

Response:
def parse_arguments(args, clone_list):
    """
    Makes parsing arguments a function.
    """
    returned_string=""
    host_number = args.host
    if args.show_list:
        print(generate_host_string(clone_list, "Available hosts: "))
        exit()
    if args.decrypt:
        for i in args.files:
            print(decrypt_files(i))
            exit()
    if args.files:
        for i in args.files:
            if args.limit_size:
                if args.host == host_number and host_number is not None:
                    if not check_max_filesize(i, clone_list[host_number][3]):
                        host_number = None
                for n, host in enumerate(clone_list):
                    if not check_max_filesize(i, host[3]):
                        clone_list[n] = None
                if not clone_list:
                    print('None of the clones is able to support so big file.')
            if args.no_cloudflare:
                if args.host == host_number and host_number is not None and not clone_list[host_number][4]:
                    print("This host uses Cloudflare, please choose different host.")
                    exit(1)
                else:
                    for n, host in enumerate(clone_list):
                        if not host[4]:
                            clone_list[n] = None
            clone_list = list(filter(None, clone_list))
            if host_number is None or args.host != host_number:
                host_number = random.randrange(0, len(clone_list))
            while True:
                try:
                    if args.encrypt:
                        returned_string = encrypt_files(clone_list[host_number], args.only_link, i)
                    else:
                        returned_string = upload_files(open(i, 'rb'), \
                              clone_list[host_number], args.only_link, i)
                    if args.only_link:
                        print(returned_string[0])
                    else:
                        print(returned_string)
                except IndexError:
                    #print('Selected server (' + clone_list[host_number][0] + ') is offline.')
                    #print('Trying other host.')
                    host_number = random.randrange(0, len(clone_list))
                    continue
                except IsADirectoryError:
                    print('limf does not support directory upload, if you want to upload ' \
                          'every file in directory use limf {}/*.'.format(i.replace('/', '')))

                if args.log:
                    with open(os.path.expanduser(args.logfile), "a+") as logfile:
                        if args.only_link:
                            logfile.write(returned_string[1])
                        else:
                            logfile.write(returned_string)
                        logfile.write("\n")
                break
    else:
        print("limf: try 'limf -h' for more information")


Instruction:
How would you explain what the following Python 3 function does
def upload_files(selected_file, selected_host, only_link, file_name):
    try:
        answer = requests.post(
            url=selected_host[0]+"upload.php",
            files={'files[]':selected_file})
        file_name_1 = re.findall(r'"url": *"((h.+\/){0,1}(.+?))"[,\}]', \
            answer.text.replace("\\", ""))[0][2]
        if only_link:
            return [selected_host[1]+file_name_1, "{}: {}{}".format(file_name, selected_host[1], file_name_1)]
        else:
            return "{}: {}{}".format(file_name, selected_host[1], file_name_1)
    except requests.exceptions.ConnectionError:
        print(file_name + ' couldn\'t be uploaded to ' + selected_host[0])

Response: Uploads selected file to the host


Instruction:
Implement a Python 3 function for
serving the Swagger UI page

Response:
def swagger_ui_template_view(request):
    """
    Serves Swagger UI page, default Swagger UI config is used but you can
    override the callable that generates the `<script>` tag by setting
    `cornice_swagger.swagger_ui_script_generator` in pyramid config, it defaults
    to 'cornice_swagger.views:swagger_ui_script_template'

    :param request:
    :return:
    """
    script_generator = request.registry.settings.get(
        'cornice_swagger.swagger_ui_script_generator',
        'cornice_swagger.views:swagger_ui_script_template')
    package, callable = script_generator.split(':')
    imported_package = importlib.import_module(package)
    script_callable = getattr(imported_package, callable)
    template = pkg_resources.resource_string(
        'cornice_swagger', 'templates/index.html').decode('utf8')

    html = Template(template).safe_substitute(
        ui_css_url=ui_css_url,
        ui_js_bundle_url=ui_js_bundle_url,
        ui_js_standalone_url=ui_js_standalone_url,
        swagger_ui_script=script_callable(request),
    )
    return Response(html)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def open_api_json_view(request):
    doc = cornice_swagger.CorniceSwagger(
        cornice.service.get_services(), pyramid_registry=request.registry)
    kwargs = request.registry.settings['cornice_swagger.spec_kwargs']
    my_spec = doc.generate(**kwargs)
    return my_spec

Response: Generates JSON representation of the current object


Instruction:
Write a Python 3 script to
generate the script code that bootstraps Swagger UI into the index template

Response:
def swagger_ui_script_template(request, **kwargs):
    """
    :param request:
    :return:

    Generates the <script> code that bootstraps Swagger UI, it will be injected
    into index template
    """
    swagger_spec_url = request.route_url('cornice_swagger.open_api_path')
    template = pkg_resources.resource_string(
        'cornice_swagger',
        'templates/index_script_template.html'
    ).decode('utf8')
    return Template(template).safe_substitute(
        swagger_spec_url=swagger_spec_url,
    )


Instruction:
Make a summary of the following Python 3 code
def set_value(request):

        key = request.matchdict['key']
        _VALUES[key] = request.json_body
        return _VALUES.get(key)

Response: Set the value and returns True or False.


Instruction:
Can you generate the documentation for the following Python 3 function
def from_schema(self, schema_node, base_name=None):
        return self._ref_recursive(self.type_converter(schema_node), self.ref, base_name)

Response: Creates a Swagger definition from a colander schema.


Instruction:
Explain what the following Python 3 code does
def _ref_recursive(self, schema, depth, base_name=None):

        if depth == 0:
            return schema

        if schema['type'] != 'object':
            return schema

        name = base_name or schema['title']

        pointer = self.json_pointer + name
        for child_name, child in schema.get('properties', {}).items():
            schema['properties'][child_name] = self._ref_recursive(child, depth-1)

        self.definition_registry[name] = schema

        return {'$ref': pointer}

Response: Recursively references a nested schema into several definitions using JSON pointers.


Instruction:
Write a Python 3 script for
creating a list of Swagger parameters from a colander request schema.

Response:
def from_schema(self, schema_node):
        """
        Creates a list of Swagger params from a colander request schema.

        :param schema_node:
            Request schema to be transformed into Swagger.
        :param validators:
            Validators used in colander with the schema.

        :rtype: list
        :returns: List of Swagger parameters.
        """

        params = []

        for param_schema in schema_node.children:
            location = param_schema.name
            if location is 'body':
                name = param_schema.__class__.__name__
                if name == 'body':
                    name = schema_node.__class__.__name__ + 'Body'
                param = self.parameter_converter(location,
                                                 param_schema)
                param['name'] = name
                if self.ref:
                    param = self._ref(param)
                params.append(param)

            elif location in (('path', 'header', 'headers', 'querystring', 'GET')):
                for node_schema in param_schema.children:
                    param = self.parameter_converter(location, node_schema)
                    if self.ref:
                        param = self._ref(param)
                    params.append(param)

        return params


Instruction:
Create a Python 3 function for
creating a list of Swagger path params from a cornice service path.

Response:
def from_path(self, path):
        """
        Create a list of Swagger path params from a cornice service path.

        :type path: string
        :rtype: list
        """
        path_components = path.split('/')
        param_names = [comp[1:-1] for comp in path_components
                       if comp.startswith('{') and comp.endswith('}')]

        params = []
        for name in param_names:
            param_schema = colander.SchemaNode(colander.String(), name=name)
            param = self.parameter_converter('path', param_schema)
            if self.ref:
                param = self._ref(param)
            params.append(param)

        return params


Instruction:
How would you implement a function in Python 3 that
returns a JSON pointer to the original parameter definition.

Response:
def _ref(self, param, base_name=None):
        """
        Store a parameter schema and return a reference to it.

        :param schema:
            Swagger parameter definition.
        :param base_name:
            Name that should be used for the reference.

        :rtype: dict
        :returns: JSON pointer to the original parameter definition.
        """

        name = base_name or param.get('title', '') or param.get('name', '')

        pointer = self.json_pointer + name
        self.parameter_registry[name] = param

        return {'$ref': pointer}


Instruction:
Given the following Python 3 function, write the documentation
def from_schema_mapping(self, schema_mapping):

        responses = {}

        for status, response_schema in schema_mapping.items():

            response = {}
            if response_schema.description:
                response['description'] = response_schema.description
            else:
                raise CorniceSwaggerException('Responses must have a description.')

            for field_schema in response_schema.children:
                location = field_schema.name

                if location == 'body':
                    title = field_schema.__class__.__name__
                    if title == 'body':
                        title = response_schema.__class__.__name__ + 'Body'
                    field_schema.title = title
                    response['schema'] = self.definitions.from_schema(field_schema)

                elif location in ('header', 'headers'):
                    header_schema = self.type_converter(field_schema)
                    headers = header_schema.get('properties')
                    if headers:
                        # Response headers doesn't accept titles
                        for header in headers.values():
                            header.pop('title')

                        response['headers'] = headers

            pointer = response_schema.__class__.__name__
            if self.ref:
                response = self._ref(response, pointer)
            responses[status] = response

        return responses

Response: Creates a Swagger response object from a dict of response schemas.


Instruction:
Given the following Python 3 function, write the documentation
def _ref(self, resp, base_name=None):

        name = base_name or resp.get('title', '') or resp.get('name', '')

        pointer = self.json_pointer + name
        self.response_registry[name] = resp

        return {'$ref': pointer}

Response: Store a response schema and return a reference to it.


Instruction:
Can you generate the documentation for the following Python 3 function
def generate(self, title=None, version=None, base_path=None,
                 info=None, swagger=None, **kwargs):
        """Generate a Swagger 2.0 documentation. Keyword arguments may be used
        to provide additional information to build methods as such ignores.

        :param title:
            The name presented on the swagger document.
        :param version:
            The version of the API presented on the swagger document.
        :param base_path:
            The path that all requests to the API must refer to.
        :param info:
            Swagger info field.
        :param swagger:
            Extra fields that should be provided on the swagger documentation.

        :rtype: dict
        :returns: Full OpenAPI/Swagger compliant specification for the application.
        """
        title = title or self.api_title
        version = version or self.api_version
        info = info or self.swagger.get('info', {})
        swagger = swagger or self.swagger
        base_path = base_path or self.base_path

        swagger = swagger.copy()
        info.update(title=title, version=version)
        swagger.update(swagger='2.0', info=info, basePath=base_path)

        paths, tags = self._build_paths()

        # Update the provided tags with the extracted ones preserving order
        if tags:
            swagger.setdefault('tags', [])
            tag_names = {t['name'] for t in swagger['tags']}
            for tag in tags:
                if tag['name'] not in tag_names:
                    swagger['tags'].append(tag)

        # Create/Update swagger sections with extracted values where not provided
        if paths:
            swagger.setdefault('paths', {})
            merge_dicts(swagger['paths'], paths)

        definitions = self.definitions.definition_registry
        if definitions:
            swagger.setdefault('definitions', {})
            merge_dicts(swagger['definitions'], definitions)

        parameters = self.parameters.parameter_registry
        if parameters:
            swagger.setdefault('parameters', {})
            merge_dicts(swagger['parameters'], parameters)

        responses = self.responses.response_registry
        if responses:
            swagger.setdefault('responses', {})
            merge_dicts(swagger['responses'], responses)

        return swagger

Response: Generate a full OpenAPI 2. 0 documentation.


Instruction:
Implement a Python 3 function for
building the Swagger paths and tags attributes from cornice service and method.

Response:
def _build_paths(self):
        """
        Build the Swagger "paths" and "tags" attributes from cornice service
        definitions.
        """
        paths = {}
        tags = []

        for service in self.services:
            path, path_obj = self._extract_path_from_service(service)

            service_tags = getattr(service, 'tags', [])
            self._check_tags(service_tags)
            tags = self._get_tags(tags, service_tags)

            for method, view, args in service.definitions:

                if method.lower() in map(str.lower, self.ignore_methods):
                    continue

                op = self._extract_operation_from_view(view, args)

                if any(ctype in op.get('consumes', []) for ctype in self.ignore_ctypes):
                    continue

                # XXX: Swagger doesn't support different schemas for for a same method
                # with different ctypes as cornice. If this happens, you may ignore one
                # content-type from the documentation otherwise we raise an Exception
                # Related to https://github.com/OAI/OpenAPI-Specification/issues/146
                previous_definition = path_obj.get(method.lower())
                if previous_definition:
                    raise CorniceSwaggerException(("Swagger doesn't support multiple "
                                                   "views for a same method. You may "
                                                   "ignore one."))

                # If tag not defined and a default tag is provided
                if 'tags' not in op and self.default_tags:
                    if callable(self.default_tags):
                        op['tags'] = self.default_tags(service, method)
                    else:
                        op['tags'] = self.default_tags

                op_tags = op.get('tags', [])
                self._check_tags(op_tags)

                # Add service tags
                if service_tags:
                    new_tags = service_tags + op_tags
                    op['tags'] = list(OrderedDict.fromkeys(new_tags))

                # Add method tags to root tags
                tags = self._get_tags(tags, op_tags)

                # If operation id is not defined and a default generator is provided
                if 'operationId' not in op and self.default_op_ids:
                    if not callable(self.default_op_ids):
                        raise CorniceSwaggerException('default_op_id should be a callable.')
                    op['operationId'] = self.default_op_ids(service, method)

                # If security options not defined and default is provided
                if 'security' not in op and self.default_security:
                    if callable(self.default_security):
                        op['security'] = self.default_security(service, method)
                    else:
                        op['security'] = self.default_security

                if not isinstance(op.get('security', []), list):
                    raise CorniceSwaggerException('security should be a list or callable')

                path_obj[method.lower()] = op
            paths[path] = path_obj

        return paths, tags


Instruction:
Can you tell what is the following Python 3 function doing
def _extract_path_from_service(self, service):

        path_obj = {}
        path = service.path
        route_name = getattr(service, 'pyramid_route', None)
        # handle services that don't create fresh routes,
        # we still need the paths so we need to grab pyramid introspector to
        # extract that information
        if route_name:
            # avoid failure if someone forgets to pass registry
            registry = self.pyramid_registry or get_current_registry()
            route_intr = registry.introspector.get('routes', route_name)
            if route_intr:
                path = route_intr['pattern']
            else:
                msg = 'Route `{}` is not found by ' \
                      'pyramid introspector'.format(route_name)
                raise ValueError(msg)

        # handle traverse and subpath as regular parameters
        # docs.pylonsproject.org/projects/pyramid/en/latest/narr/hybrid.html
        for subpath_marker in ('*subpath', '*traverse'):
            path = path.replace(subpath_marker, '{subpath}')

        # Extract path parameters
        parameters = self.parameters.from_path(path)
        if parameters:
            path_obj['parameters'] = parameters

        return path, path_obj

Response: Extract path object and its parameters from service definitions.


Instruction:
How would you explain what the following Python 3 function does
def _extract_operation_from_view(self, view, args):

        op = {
            'responses': {
                'default': {
                    'description': 'UNDOCUMENTED RESPONSE'
                }
            },
        }

        # If 'produces' are not defined in the view, try get from renderers
        renderer = args.get('renderer', '')

        if "json" in renderer:  # allows for "json" or "simplejson"
            produces = ['application/json']
        elif renderer == 'xml':
            produces = ['text/xml']
        else:
            produces = None

        if produces:
            op.setdefault('produces', produces)

        # Get explicit accepted content-types
        consumes = args.get('content_type')

        if consumes is not None:
            # convert to a list, if it's not yet one
            consumes = to_list(consumes)

            # It is possible to add callables for content_type, so we have to
            # to filter those out, since we cannot evaluate those here.
            consumes = [x for x in consumes if not callable(x)]
            op['consumes'] = consumes

        # Get parameters from view schema
        is_colander = self._is_colander_schema(args)
        if is_colander:
            schema = self._extract_transform_colander_schema(args)
            parameters = self.parameters.from_schema(schema)
        else:
            # Bail out for now
            parameters = None
        if parameters:
            op['parameters'] = parameters

        # Get summary from docstring
        if isinstance(view, six.string_types):
            if 'klass' in args:
                ob = args['klass']
                view_ = getattr(ob, view.lower())
                docstring = trim(view_.__doc__)
        else:
            docstring = str(trim(view.__doc__))

        if docstring and self.summary_docstrings:
            op['summary'] = docstring

        # Get response definitions
        if 'response_schemas' in args:
            op['responses'] = self.responses.from_schema_mapping(args['response_schemas'])

        # Get response tags
        if 'tags' in args:
            op['tags'] = args['tags']

        # Get response operationId
        if 'operation_id' in args:
            op['operationId'] = args['operation_id']

        # Get security policies
        if 'api_security' in args:
            op['security'] = args['api_security']

        return op

Response: Extracts the swagger operation details from colander view definitions.


Instruction:
Implement a Python 3 function for
extracting schema from view args and transform it using the pipeline of schema transformers

Response:
def _extract_transform_colander_schema(self, args):
        """
        Extract schema from view args and transform it using
        the pipeline of schema transformers

        :param args:
            Arguments from the view decorator.

        :rtype: colander.MappingSchema()
        :returns: View schema cloned and transformed
        """

        schema = args.get('schema', colander.MappingSchema())
        if not isinstance(schema, colander.Schema):
            schema = schema()
        schema = schema.clone()
        for transformer in self.schema_transformers:
            schema = transformer(schema, args)
        return schema


Instruction:
Create a Python 3 function for
creating arguments and parses user input

Response:
def main():
    """Creates arguments and parses user input"""
    parser = argparse.ArgumentParser(
        description=_('Uploads selected file to working pomf.se clone'))
    parser.add_argument('files', metavar='file', nargs='*', type=str,
                        help=_('Files to upload'))
    parser.add_argument('-c', metavar='host_number', type=int,
                        dest='host', default=None,
                        help=_('The number (0-n) of the selected host (default is random)'))
    parser.add_argument('-l', dest='only_link', action='store_const',
                        const=True, default=False,
                        help=_('Changes output to just link to the file'))
    parser.add_argument('-e', dest='encrypt', action='store_const',
                        const=True, default=False,
                        help=_('Encrypts then uploads the files.'))
    parser.add_argument('-d', dest='decrypt', action='store_const',
                        const=True, default=False,
                        help=_('Decrypts files from links with encrypted files'))
    parser.add_argument('-j', dest="local_list",
                        default=False, help=_('Path to a local list file'))
    parser.add_argument('-s', dest="show_list", action='store_const',
                        const=True, default=False,
                        help=_('Show the host list (will not upload your files when called)'))
    parser.add_argument('-m', dest='limit_size', action='store_const',
                        const=True, default=False,
                        help=_('Do not upload file if it exceeds the certain host limit'))
    parser.add_argument('-nc', dest='no_cloudflare', action='store_const',
                        const=True, default=False,
                        help=_('Do not use hosts which use Cloudflare.'))
    parser.add_argument('--log-file', metavar="LOGFILE", dest="logfile",
                        default="~/limf.log",
                        help=_("The location of log file"))
    parser.add_argument('--log', dest='log', action="store_const",
                        const=True, default=False,
                        help=_("Enables the logging feature, default logfile is ~/limf.log"))
    args = parser.parse_args()
    try:
        if args.local_list:
            clone_list = retrieve_local_host_list(args.local_list)
        else:
            clone_list = retrieve_online_host_list()
        if len(min(clone_list, key=len)) < 5 and (args.limit_size or args.no_cloudflare):
            print(_("For newer options, please update your host_list."))
            exit()
        if args.host and not(0 <= args.host < len(clone_list)):
            print(generate_host_string(clone_list))
            exit()

        parse_arguments(args, clone_list)
    except FileNotFoundError:
        print(_('Plese enter valid file.'))


Instruction:
Create a Python 3 function for
converting a node schema into a parameter object.

Response:
def convert(self, schema_node, definition_handler):
        """
        Convert node schema into a parameter object.
        """

        converted = {
            'name': schema_node.name,
            'in': self._in,
            'required': schema_node.required
        }
        if schema_node.description:
            converted['description'] = schema_node.description

        if schema_node.default:
            converted['default'] = schema_node.default

        schema = definition_handler(schema_node)
        # Parameters shouldn't have a title
        schema.pop('title', None)
        converted.update(schema)

        if schema.get('type') == 'array':
            converted['items'] = {'type': schema['items']['type']}

        return converted


Instruction:
Can you generate a brief explanation for the following Python 3 code
def cornice_enable_openapi_view(
        config,
        api_path='/api-explorer/swagger.json',
        permission=NO_PERMISSION_REQUIRED,
        route_factory=None, **kwargs):
    """
    :param config:
        Pyramid configurator object
    :param api_path:
        where to expose swagger JSON definition view
    :param permission:
        pyramid permission for those views
    :param route_factory:
        factory for context object for those routes
    :param kwargs:
        kwargs that will be passed to CorniceSwagger's `generate()`

    This registers and configures the view that serves api definitions
    """
    config.registry.settings['cornice_swagger.spec_kwargs'] = kwargs
    config.add_route('cornice_swagger.open_api_path', api_path,
                     factory=route_factory)
    config.add_view('cornice_swagger.views.open_api_json_view',
                    renderer='json', permission=permission,
                    route_name='cornice_swagger.open_api_path')

Response: Enables the OpenAPI view for the given resource.


Instruction:
Can you tell what is the following Python 3 function doing
def cornice_enable_openapi_explorer(
        config,
        api_explorer_path='/api-explorer',
        permission=NO_PERMISSION_REQUIRED,
        route_factory=None,
        **kwargs):
    """
    :param config:
        Pyramid configurator object
    :param api_explorer_path:
        where to expose Swagger UI interface view
    :param permission:
        pyramid permission for those views
    :param route_factory:
        factory for context object for those routes

    This registers and configures the view that serves api explorer
    """
    config.add_route('cornice_swagger.api_explorer_path', api_explorer_path,
                     factory=route_factory)
    config.add_view('cornice_swagger.views.swagger_ui_template_view',
                    permission=permission,
                    route_name='cornice_swagger.api_explorer_path')

Response: Enable the OpenAPI explorer for the given resource.


Instruction:
Write a Python 3 function for
trimming the given docstring and return a new string.

Response:
def trim(docstring):
    """
    Remove the tabs to spaces, and remove the extra spaces / tabs that are in
    front of the text in docstrings.

    Implementation taken from http://www.python.org/dev/peps/pep-0257/
    """
    if not docstring:
        return ''
    # Convert tabs to spaces (following the normal Python rules)
    # and split into a list of lines:
    lines = six.u(docstring).expandtabs().splitlines()
    lines = [line.strip() for line in lines]
    res = six.u('\n').join(lines)
    return res


Instruction:
Implement a Python 3 function for
merging a dict into a recursively without overwriting values.

Response:
def merge_dicts(base, changes):
    """Merge b into a recursively, without overwriting values.

    :param base: the dict that will be altered.
    :param changes: changes to update base.
    """
    for k, v in changes.items():
        if isinstance(v, dict):
            merge_dicts(base.setdefault(k, {}), v)
        else:
            base.setdefault(k, v)


Instruction:
Write a Python 3 function that can
create a viewset method for the provided transition_name

Response:
def get_transition_viewset_method(transition_name, **kwargs):
    '''
    Create a viewset method for the provided `transition_name`
    '''
    @detail_route(methods=['post'], **kwargs)
    def inner_func(self, request, pk=None, **kwargs):
        object = self.get_object()
        transition_method = getattr(object, transition_name)

        transition_method(by=self.request.user)

        if self.save_after_transition:
            object.save()

        serializer = self.get_serializer(object)
        return Response(serializer.data)

    return inner_func


Instruction:
Explain what the following Python 3 code does
def get_viewset_transition_action_mixin(model, **kwargs):
    '''
    Find all transitions defined on `model`, then create a corresponding
    viewset action method for each and apply it to `Mixin`. Finally, return
    `Mixin`
    '''
    instance = model()

    class Mixin(object):
        save_after_transition = True

    transitions = instance.get_all_status_transitions()
    transition_names = set(x.name for x in transitions)
    for transition_name in transition_names:
        setattr(
            Mixin,
            transition_name,
            get_transition_viewset_method(transition_name, **kwargs)
        )

    return Mixin

Response: Returns a Mixin class that will create all transitions defined on the model and apply it to it.


Instruction:
Write a Python 3 script to
refresh the project from the original cookiecutter template.

Response:
def fresh_cookies(ctx, mold=''):
    """Refresh the project from the original cookiecutter template."""
    mold = mold or "https://github.com/Springerle/py-generic-project.git"  # TODO: URL from config
    tmpdir = os.path.join(tempfile.gettempdir(), "cc-upgrade-pygments-markdown-lexer")

    if os.path.isdir('.git'):
        # TODO: Ensure there are no local unstashed changes
        pass

    # Make a copy of the new mold version
    if os.path.isdir(tmpdir):
        shutil.rmtree(tmpdir)
    if os.path.exists(mold):
        shutil.copytree(mold, tmpdir, ignore=shutil.ignore_patterns(
            ".git", ".svn", "*~",
        ))
    else:
        ctx.run("git clone {} {}".format(mold, tmpdir))

    # Copy recorded "cookiecutter.json" into mold
    shutil.copy2("project.d/cookiecutter.json", tmpdir)

    with pushd('..'):
        ctx.run("cookiecutter --no-input {}".format(tmpdir))
    if os.path.exists('.git'):
        ctx.run("git status")


Instruction:
Can you tell what is the following Python 3 function doing
def ci(ctx):
    opts = ['']

    # 'tox' makes no sense in Travis
    if os.environ.get('TRAVIS', '').lower() == 'true':
        opts += ['test.pytest']
    else:
        opts += ['test.tox']

    ctx.run("invoke --echo --pty clean --all build --docs check --reports{}".format(' '.join(opts)))

Response: Perform continuous integration tasks.


Instruction:
How would you implement a function in Python 3 that
returns project s metadata as a dict.

Response:
def _build_metadata(): # pylint: disable=too-many-locals, too-many-branches
    "Return project's metadata as a dict."
    # Handle metadata in package source
    expected_keys = ('url', 'version', 'license', 'author', 'author_email', 'long_description', 'keywords')
    metadata = {}
    with io.open(srcfile('src', package_name, '__init__.py'), encoding='utf-8') as handle:
        pkg_init = handle.read()
        # Get default long description from docstring
        metadata['long_description'] = re.search(r'^"""(.+?)^"""$', pkg_init, re.DOTALL|re.MULTILINE).group(1)
        for line in pkg_init.splitlines():
            match = re.match(r"""^__({0})__ += (?P<q>['"])(.+?)(?P=q)$""".format('|'.join(expected_keys)), line)
            if match:
                metadata[match.group(1)] = match.group(3)

    if not all(i in metadata for i in expected_keys):
        raise RuntimeError("Missing or bad metadata in '{0}' package: {1}"
                           .format(name, ', '.join(sorted(set(expected_keys) - set(metadata.keys()))),))

    text = metadata['long_description'].strip()
    if text:
        metadata['description'], text = text.split('.', 1)
        metadata['description'] = ' '.join(metadata['description'].split()).strip() + '.' # normalize whitespace
        metadata['long_description'] = textwrap.dedent(text).strip()
    metadata['keywords'] = metadata['keywords'].replace(',', ' ').strip().split()

    # Load requirements files
    requirements_files = dict(
        install = 'requirements.txt',
        setup = 'setup-requirements.txt',
        test = 'test-requirements.txt',
    )
    requires = {}
    for key, filename in requirements_files.items():
        requires[key] = []
        if os.path.exists(srcfile(filename)):
            with io.open(srcfile(filename), encoding='utf-8') as handle:
                for line in handle:
                    line = line.strip()
                    if line and not line.startswith('#'):
                        if any(line.startswith(i) for i in ('-e', 'http://', 'https://')):
                            line = line.split('#egg=')[1]
                        requires[key].append(line)
    if not any('pytest' == re.split('[\t ,<=>]', i.lower())[0] for i in requires['test']):
        requires['test'].append('pytest') # add missing requirement

    # CLI entry points
    console_scripts = []
    for path, dirs, files in os.walk(srcfile('src', package_name)):
        dirs = [i for i in dirs if not i.startswith('.')]
        if '__main__.py' in files:
            path = path[len(srcfile('src') + os.sep):]
            appname = path.split(os.sep)[-1]
            with io.open(srcfile('src', path, '__main__.py'), encoding='utf-8') as handle:
                for line in handle.readlines():
                    match = re.match(r"""^__app_name__ += (?P<q>['"])(.+?)(?P=q)$""", line)
                    if match:
                        appname = match.group(2)
            console_scripts.append('{0} = {1}.__main__:cli'.format(appname, path.replace(os.sep, '.')))

    # Add some common files to EGG-INFO
    candidate_files = [
        'LICENSE', 'NOTICE',
        'README', 'README.md', 'README.rst', 'README.txt',
        'CHANGES', 'CHANGELOG', 'debian/changelog',
    ]
    data_files = defaultdict(list)
    for filename in candidate_files:
        if os.path.exists(srcfile(filename)):
            data_files['EGG-INFO'].append(filename)

    # Complete project metadata
    classifiers = []
    for classifiers_txt in ('classifiers.txt', 'project.d/classifiers.txt'):
        classifiers_txt = srcfile(classifiers_txt)
        if os.path.exists(classifiers_txt):
            with io.open(classifiers_txt, encoding='utf-8') as handle:
                classifiers = [i.strip() for i in handle if i.strip() and not i.startswith('#')]
            break
    entry_points.setdefault('console_scripts', []).extend(console_scripts)

    metadata.update(dict(
        name = name,
        package_dir = {'': 'src'},
        packages = find_packages(srcfile('src'), exclude=['tests']),
        data_files = data_files.items(),
        zip_safe = False,
        include_package_data = True,
        install_requires = requires['install'],
        setup_requires = requires['setup'],
        tests_require =  requires['test'],
        classifiers = classifiers,
        cmdclass = dict(
            test = PyTest,
        ),
        entry_points = entry_points,
    ))
    return metadata


Instruction:
Given the following Python 3 function, write the documentation
def py_hash(key, num_buckets):
    b, j = -1, 0

    if num_buckets < 1:
        raise ValueError('num_buckets must be a positive number')

    while j < num_buckets:
        b = int(j)
        key = ((key * long(2862933555777941757)) + 1) & 0xffffffffffffffff
        j = float(b + 1) * (float(1 << 31) / float((key >> 33) + 1))

    return int(b)

Response: Generate a number in the range [ 0 num_buckets ).


Instruction:
Here you have a function in Python 3, explain what it does
def setup(app):
    lexer = MarkdownLexer()
    for alias in lexer.aliases:
        app.add_lexer(alias, lexer)

    return dict(version=__version__)

Response: Setup Sphinx extension API.


Instruction:
Here you have a function in Python 3, explain what it does
def load(self):
        ret = {}

        # Read the mdstat file
        with open(self.get_path(), 'r') as f:
            # lines is a list of line (with \n)
            lines = f.readlines()

        # First line: get the personalities
        # The "Personalities" line tells you what RAID level the kernel currently supports.
        # This can be changed by either changing the raid modules or recompiling the kernel.
        # Possible personalities include: [raid0] [raid1] [raid4] [raid5] [raid6] [linear] [multipath] [faulty]
        ret['personalities'] = self.get_personalities(lines[0])

        # Second to last before line: Array definition
        ret['arrays'] = self.get_arrays(lines[1:-1], ret['personalities'])

        # Save the file content as it for the __str__ method
        self.content = reduce(lambda x, y: x + y, lines)

        return ret

Response: Load the stats from the file and return a dict of stats.


Instruction:
Can you write a function in Python 3 where it
returns a list of personalities readed from the input line.

Response:
def get_personalities(self, line):
        """Return a list of personalities readed from the input line."""
        return [split('\W+', i)[1] for i in line.split(':')[1].split(' ') if i.startswith('[')]


Instruction:
Implement a Python 3 function for
returning a dict of arrays.

Response:
def get_arrays(self, lines, personalities=[]):
        """Return a dict of arrays."""
        ret = {}

        i = 0
        while i < len(lines):
            try:
                # First array line: get the md device
                md_device = self.get_md_device_name(lines[i])
            except IndexError:
                # No array detected
                pass
            else:
                # Array detected
                if md_device is not None:
                    # md device line
                    ret[md_device] = self.get_md_device(lines[i], personalities)
                    # md config/status line
                    i += 1
                    ret[md_device].update(self.get_md_status(lines[i]))
            i += 1

        return ret


Instruction:
Write a Python 3 script for
returning a dict of md device define in the line.

Response:
def get_md_device(self, line, personalities=[]):
        """Return a dict of md device define in the line."""
        ret = {}

        splitted = split('\W+', line)
        # Raid status
        # Active or 'started'. An inactive array is usually faulty.
        # Stopped arrays aren't visible here.
        ret['status'] = splitted[1]
        if splitted[2] in personalities:
            # Raid type (ex: RAID5)
            ret['type'] = splitted[2]
            # Array's components
            ret['components'] = self.get_components(line, with_type=True)
        else:
            # Raid type (ex: RAID5)
            ret['type'] = None
            # Array's components
            ret['components'] = self.get_components(line, with_type=False)

        return ret


Instruction:
Write a Python 3 function that can
return a dict of md status define in the line.

Response:
def get_md_status(self, line):
        """Return a dict of md status define in the line."""
        ret = {}

        splitted = split('\W+', line)
        if len(splitted) < 7:
            ret['available'] = None
            ret['used'] = None
            ret['config'] = None
        else:
            # The final 2 entries on this line: [n/m] [UUUU_]
            # [n/m] means that ideally the array would have n devices however, currently, m devices are in use.
            # Obviously when m >= n then things are good.
            ret['available'] = splitted[-4]
            ret['used'] = splitted[-3]
            # [UUUU_] represents the status of each device, either U for up or _ for down.
            ret['config'] = splitted[-2]

        return ret


Instruction:
How would you code a function in Python 3 to
return a dict of components in the line.

Response:
def get_components(self, line, with_type=True):
        """Return a dict of components in the line.

        key: device name (ex: 'sdc1')
        value: device role number
        """
        ret = {}

        # Ignore (F) (see test 08)
        line2 = reduce(lambda x, y: x + y, split('\(.+\)', line))
        if with_type:
            splitted = split('\W+', line2)[3:]
        else:
            splitted = split('\W+', line2)[2:]
        ret = dict(zip(splitted[0::2], splitted[1::2]))

        return ret


Instruction:
Write a Python 3 script to
register signal receivers which send events.

Response:
def register_receivers(app, config):
    """Register signal receivers which send events."""
    for event_name, event_config in config.items():
        event_builders = [
            obj_or_import_string(func)
            for func in event_config.get('event_builders', [])
        ]

        signal = obj_or_import_string(event_config['signal'])
        signal.connect(
            EventEmmiter(event_name, event_builders), sender=app, weak=False
        )


Instruction:
Make a summary of the following Python 3 code
def check(self, query):
        if query.get_type() != Keyword.DELETE:
            return Ok(True)

        return Err("Delete queries are forbidden.")

Response: Check if the query is for DELETE type.


Instruction:
Can you tell what is the following Python 3 function doing
def set_scheduled(self):
        with self._idle_lock:
            if self._idle:
                self._idle = False
                return True
        return False

Response: Returns True if state was successfully changed from idle to scheduled.


Instruction:
How would you explain what the following Python 3 function does
def check(self, query):
        if query.get_type() not in {Keyword.SELECT, Keyword.DELETE}:
            # Only select and delete queries deal with time durations
            # All others are not affected by this rule. Bailing out.
            return Ok(True)

        datapoints = query.get_datapoints()
        if datapoints <= self.max_datapoints:
            return Ok(True)

        return Err(("Expecting {} datapoints from that query, which is above the threshold! "
                    "Set a date range (e.g. where time > now() - 24h), "
                    "increase grouping (e.g. group by time(24h) "
                    "or limit the number of datapoints (e.g. limit 100)").format(datapoints))

Response: Check if the given query is valid.


Instruction:
Can you create a Python 3 function that
searches for the oldest event timestamp.

Response:
def _get_oldest_event_timestamp(self):
        """Search for the oldest event timestamp."""
        # Retrieve the oldest event in order to start aggregation
        # from there
        query_events = Search(
            using=self.client,
            index=self.event_index
        )[0:1].sort(
            {'timestamp': {'order': 'asc'}}
        )
        result = query_events.execute()
        # There might not be any events yet if the first event have been
        # indexed but the indices have not been refreshed yet.
        if len(result) == 0:
            return None
        return parser.parse(result[0]['timestamp'])


Instruction:
Write a Python 3 function for
getting the last aggregation date.

Response:
def get_bookmark(self):
        """Get last aggregation date."""
        if not Index(self.aggregation_alias,
                     using=self.client).exists():
            if not Index(self.event_index,
                         using=self.client).exists():
                return datetime.date.today()
            return self._get_oldest_event_timestamp()

        # retrieve the oldest bookmark
        query_bookmark = Search(
            using=self.client,
            index=self.aggregation_alias,
            doc_type=self.bookmark_doc_type
        )[0:1].sort(
            {'date': {'order': 'desc'}}
        )
        bookmarks = query_bookmark.execute()
        # if no bookmark is found but the index exist, the bookmark was somehow
        # lost or never written, so restart from the beginning
        if len(bookmarks) == 0:
            return self._get_oldest_event_timestamp()

        # change it to doc_id_suffix
        bookmark = datetime.datetime.strptime(bookmarks[0].date,
                                              self.doc_id_suffix)
        return bookmark


Instruction:
Write a Python 3 script to
set bookmark for starting next aggregation.

Response:
def set_bookmark(self):
        """Set bookmark for starting next aggregation."""
        def _success_date():
            bookmark = {
                'date': self.new_bookmark or datetime.datetime.utcnow().
                strftime(self.doc_id_suffix)
            }

            yield dict(_index=self.last_index_written,
                       _type=self.bookmark_doc_type,
                       _source=bookmark)
        if self.last_index_written:
            bulk(self.client,
                 _success_date(),
                 stats_only=True)


Instruction:
Create a Python 3 function for
formatting the datetime to the closest aggregation interval.

Response:
def _format_range_dt(self, d):
        """Format range filter datetime to the closest aggregation interval."""
        if not isinstance(d, six.string_types):
            d = d.isoformat()
        return '{0}||/{1}'.format(
            d, self.dt_rounding_map[self.aggregation_interval])


Instruction:
How would you code a function in Python 3 to
aggregate and return an iterator over the event index entries.

Response:
def agg_iter(self, lower_limit=None, upper_limit=None):
        """Aggregate and return dictionary to be indexed in ES."""
        lower_limit = lower_limit or self.get_bookmark().isoformat()
        upper_limit = upper_limit or (
            datetime.datetime.utcnow().replace(microsecond=0).isoformat())
        aggregation_data = {}

        self.agg_query = Search(using=self.client,
                                index=self.event_index).\
            filter('range', timestamp={
                'gte': self._format_range_dt(lower_limit),
                'lte': self._format_range_dt(upper_limit)})

        # apply query modifiers
        for modifier in self.query_modifiers:
            self.agg_query = modifier(self.agg_query)

        hist = self.agg_query.aggs.bucket(
            'histogram',
            'date_histogram',
            field='timestamp',
            interval=self.aggregation_interval
        )
        terms = hist.bucket(
            'terms', 'terms', field=self.aggregation_field, size=0
        )
        top = terms.metric(
            'top_hit', 'top_hits', size=1, sort={'timestamp': 'desc'}
        )
        for dst, (metric, src, opts) in self.metric_aggregation_fields.items():
            terms.metric(dst, metric, field=src, **opts)

        results = self.agg_query.execute()
        index_name = None
        for interval in results.aggregations['histogram'].buckets:
            interval_date = datetime.datetime.strptime(
                interval['key_as_string'], '%Y-%m-%dT%H:%M:%S')
            for aggregation in interval['terms'].buckets:
                aggregation_data['timestamp'] = interval_date.isoformat()
                aggregation_data[self.aggregation_field] = aggregation['key']
                aggregation_data['count'] = aggregation['doc_count']

                if self.metric_aggregation_fields:
                    for f in self.metric_aggregation_fields:
                        aggregation_data[f] = aggregation[f]['value']

                doc = aggregation.top_hit.hits.hits[0]['_source']
                for destination, source in self.copy_fields.items():
                    if isinstance(source, six.string_types):
                        aggregation_data[destination] = doc[source]
                    else:
                        aggregation_data[destination] = source(
                            doc,
                            aggregation_data
                        )

                index_name = 'stats-{0}-{1}'.\
                             format(self.event,
                                    interval_date.strftime(
                                        self.index_name_suffix))
                self.indices.add(index_name)
                yield dict(_id='{0}-{1}'.
                           format(aggregation['key'],
                                  interval_date.strftime(
                                      self.doc_id_suffix)),
                           _index=index_name,
                           _type=self.aggregation_doc_type,
                           _source=aggregation_data)
        self.last_index_written = index_name


Instruction:
Can you generate the documentation for the following Python 3 function
def list_bookmarks(self, start_date=None, end_date=None, limit=None):
        query = Search(
            using=self.client,
            index=self.aggregation_alias,
            doc_type=self.bookmark_doc_type
        ).sort({'date': {'order': 'desc'}})

        range_args = {}
        if start_date:
            range_args['gte'] = self._format_range_dt(
                start_date.replace(microsecond=0))
        if end_date:
            range_args['lte'] = self._format_range_dt(
                end_date.replace(microsecond=0))
        if range_args:
            query = query.filter('range', date=range_args)

        return query[0:limit].execute() if limit else query.scan()

Response: List the aggregation s bookmarks.


Instruction:
Write a Python 3 function for
parsing the data resolution of a query in seconds

Response:
def parse(self, group_by_stmt):
        """
        Extract the data resolution of a query in seconds
        E.g. "group by time(99s)" => 99

        :param group_by_stmt: A raw InfluxDB group by statement
        """
        if not group_by_stmt:
            return Resolution.MAX_RESOLUTION

        m = self.GROUP_BY_TIME_PATTERN.match(group_by_stmt)
        if not m:
            return None

        value = int(m.group(1))
        unit = m.group(2)
        resolution = self.convert_to_seconds(value, unit)

        # We can't have a higher resolution than the max resolution
        return max(resolution, Resolution.MAX_RESOLUTION)


Instruction:
Can you tell what is the following Python 3 function doing
def get(self, timeout=None):
        result = None
        try:
            result = self._result.get(True, timeout=timeout)
        except Empty:
            raise Timeout()

        if isinstance(result, Failure):
            six.reraise(*result.exc_info)
        else:
            return result

Response: Get the next entry from the cache.


Instruction:
Explain what the following Python 3 code does
def consume(self, event_type, no_ack=True, payload=True):
        assert event_type in self.events
        return current_queues.queues['stats-{}'.format(event_type)].consume(
            payload=payload)

Response: Comsume all pending events.


Instruction:
Can you create a Python 3 function that
initializes the application with the given Flask application.

Response:
def init_app(self, app,
                 entry_point_group_events='invenio_stats.events',
                 entry_point_group_aggs='invenio_stats.aggregations',
                 entry_point_group_queries='invenio_stats.queries'):
        """Flask application initialization."""
        self.init_config(app)

        state = _InvenioStatsState(
            app,
            entry_point_group_events=entry_point_group_events,
            entry_point_group_aggs=entry_point_group_aggs,
            entry_point_group_queries=entry_point_group_queries
        )
        self._state = app.extensions['invenio-stats'] = state

        if app.config['STATS_REGISTER_RECEIVERS']:
            signal_receivers = {key: value for key, value in
                                app.config.get('STATS_EVENTS', {}).items()
                                if 'signal' in value}
            register_receivers(app, signal_receivers)

        return state


Instruction:
How would you implement a function in Python 3 that
sends a message to this actor. Asynchronous fire - and - forget.

Response:
def tell(self, message, sender=no_sender):
        """ Send a message to this actor. Asynchronous fire-and-forget.

        :param message: The message to send.
        :type message: Any

        :param sender: The sender of the message. If provided it will be made
            available to the receiving actor via the :attr:`Actor.sender` attribute.
        :type sender: :class:`Actor`
        """
        if sender is not no_sender and not isinstance(sender, ActorRef):
            raise ValueError("Sender must be actor reference")

        self._cell.send_message(message, sender)


Instruction:
Can you tell what is the following Python 3 function doing
def get_anonymization_salt(ts):
    salt_key = 'stats:salt:{}'.format(ts.date().isoformat())
    salt = current_cache.get(salt_key)
    if not salt:
        salt_bytes = os.urandom(32)
        salt = b64encode(salt_bytes).decode('utf-8')
        current_cache.set(salt_key, salt, timeout=60 * 60 * 24)
    return salt

Response: Get the anonymization salt based on the event timestamp s day.


Instruction:
Can you tell what is the following Python 3 function doing
def get_user():
    return dict(
        ip_address=request.remote_addr,
        user_agent=request.user_agent.string,
        user_id=(
            current_user.get_id() if current_user.is_authenticated else None
        ),
        session_id=session.get('sid_s')
    )

Response:
User information.

    .. note::

       **Privacy note** A users IP address, user agent string, and user id
       (if logged in) is sent to a message queue, where it is stored for about
       5 minutes. The information is used to:

       - Detect robot visits from the user agent string.
       - Generate an anonymized visitor id (using a random salt per day).
       - Detect the users host contry based on the IP address.

       The information is then discarded.


Instruction:
Make a summary of the following Python 3 code
def default_permission_factory(query_name, params):
    from invenio_stats import current_stats
    if current_stats.queries[query_name].permission_factory is None:
        return AllowAllPermission
    else:
        return current_stats.queries[query_name].permission_factory(
            query_name, params
        )

Response:
Default permission factory.

    It enables by default the statistics if they don't have a dedicated
    permission factory.


Instruction:
Given the following Python 3 function, write the documentation
def load_config():
    # We start with the default config
    config = flatten(default_config.DEFAULT_CONFIG)

    # Read commandline arguments
    cli_config = flatten(parse_args())

    if "configfile" in cli_config:
        logging.info("Reading config file {}".format(cli_config['configfile']))
        configfile = parse_configfile(cli_config['configfile'])
        config = overwrite_config(config, configfile)

    # Parameters from commandline take precedence over all others
    config = overwrite_config(config, cli_config)

    # Set verbosity level
    if 'verbose' in config:
        if config['verbose'] == 1:
            logging.getLogger().setLevel(logging.INFO)
        elif config['verbose'] > 1:
            logging.getLogger().setLevel(logging.DEBUG)

    return ObjectView(config)

Response: Load settings from default config and optionally overwrite with config file and commandline parameters


Instruction:
Can you write a function in Python 3 where it
reads settings from file ridge

Response:
def parse_configfile(configfile):
    """
    Read settings from file
    :param configfile:
    """
    with open(configfile) as f:
        try:
            return yaml.safe_load(f)
        except Exception as e:
            logging.fatal("Could not load default config file: %s", e)
            exit(-1)


Instruction:
Write a Python 3 function that can
register elasticsearch templates for events and aggregations.

Response:
def register_templates():
    """Register elasticsearch templates for events."""
    event_templates = [current_stats._events_config[e]
                       ['templates']
                       for e in
                       current_stats._events_config]
    aggregation_templates = [current_stats._aggregations_config[a]
                             ['templates']
                             for a in
                             current_stats._aggregations_config]
    return event_templates + aggregation_templates


Instruction:
Create a Python 3 function to
check that the series name is not too short.

Response:
def check(self, query):
        """
        :param query:
        """
        if query.get_type() in {Keyword.LIST, Keyword.DROP}:
            series = query.series_stmt
        else:
            series = query.from_stmt

        if len(series) >= self.min_series_name_length:
            return Ok(True)

        return Err("Series name too short. Please be more precise.")


Instruction:
Write a Python 3 script to
send a message to actor and return a future holding the result.

Response:
def ask(actor, message):
    """
    Send a message to `actor` and return a :class:`Future` holding a possible
    reply.

    To receive a result, the actor MUST send a reply to `sender`.

    :param actor:
    :type actor: :class:`ActorRef`.

    :param message:
    :type message: :type: Any

    :return: A future holding the result.
    """
    sender = PromiseActorRef()
    actor.tell(message, sender)
    return sender.promise.future


Instruction:
Implement a Python 3 function for
getting a list of all queries from an URL parameter string

Response:
def get_queries(parameters):
        """
        Get a list of all queries (q=... parameters) from an URL parameter string
        :param parameters: The url parameter list
        """
        parsed_params = urlparse.parse_qs(parameters)
        if 'q' not in parsed_params:
            return []
        queries = parsed_params['q']

        # Check if only one query string is given
        # in this case make it a list
        if not isinstance(queries, list):
            queries = [queries]
        return queries


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _handle_request(self, scheme, netloc, path, headers, body=None, method="GET"):
        backend_url = "{}://{}{}".format(scheme, netloc, path)
        try:
            response = self.http_request.request(backend_url, method=method, body=body, headers=dict(headers))
            self._return_response(response)
        except Exception as e:
            body = "Invalid response from backend: '{}' Server might be busy".format(e.message)
            logging.debug(body)
            self.send_error(httplib.SERVICE_UNAVAILABLE, body)

Response: Handle the request from the backend and return the response.


Instruction:
How would you implement a function in Python 3 that
sends and log plain text error reply.

Response:
def send_error(self, code, message=None):
        """
        Send and log plain text error reply.
        :param code:
        :param message:
        """
        message = message.strip()
        self.log_error("code %d, message %s", code, message)
        self.send_response(code)
        self.send_header("Content-Type", "text/plain")
        self.send_header('Connection', 'close')
        self.end_headers()
        if message:
            self.wfile.write(message)


Instruction:
Make a summary of the following Python 3 code
def _return_response(self, response):
        self.filter_headers(response.msg)
        if "content-length" in response.msg:
            del response.msg["content-length"]

        self.send_response(response.status, response.reason)
        for header_key, header_value in response.msg.items():
            self.send_header(header_key, header_value)
        body = response.read()
        self.send_header('Content-Length', str(len(body)))
        self.end_headers()
        self.wfile.write(body)

Response: Send the response to the client.


Instruction:
Explain what the following Python 3 code does
def hash_id(iso_timestamp, msg):
    return '{0}-{1}'.format(iso_timestamp,
                            hashlib.sha1(
                                msg.get('unique_id').encode('utf-8') +
                                str(msg.get('visitor_id')).
                                encode('utf-8')).
                            hexdigest())

Response: Generate event id optimized for ES.


Instruction:
Given the following Python 3 function, write the documentation
def actionsiter(self):
        for msg in self.queue.consume():
            try:
                for preproc in self.preprocessors:
                    msg = preproc(msg)
                    if msg is None:
                        break
                if msg is None:
                    continue
                suffix = arrow.get(msg.get('timestamp')).strftime(self.suffix)
                ts = parser.parse(msg.get('timestamp'))
                # Truncate timestamp to keep only seconds. This is to improve
                # elasticsearch performances.
                ts = ts.replace(microsecond=0)
                msg['timestamp'] = ts.isoformat()
                # apply timestamp windowing in order to group events too close
                # in time
                if self.double_click_window > 0:
                    timestamp = mktime(utc.localize(ts).utctimetuple())
                    ts = ts.fromtimestamp(
                        timestamp // self.double_click_window *
                        self.double_click_window
                    )
                yield dict(
                    _id=hash_id(ts.isoformat(), msg),
                    _op_type='index',
                    _index='{0}-{1}'.format(self.index, suffix),
                    _type=self.doctype,
                    _source=msg,
                )
            except Exception:
                current_app.logger.exception(u'Error while processing event')

Response: Iterate over the messages in the queue and yield the events that were processed.


Instruction:
Write a Python 3 script to
parse the number of datapoints in a given duration in seconds.

Response:
def parse(duration_seconds, resolution_seconds=Resolution.MAX_RESOLUTION, limit=None):
        """
        num_datapoints = min(duration/resolution, limit)

        :param duration_seconds: Time duration (in seconds) for which datapoints should be returned
        :param resolution_seconds: Time interval (in seconds) between data points
        :param limit: Maximum number of datapoints to return
        """

        if not duration_seconds or duration_seconds < 0:
            return 0

        if not resolution_seconds or resolution_seconds <= 0:
            return None

        num_datapoints = duration_seconds / resolution_seconds

        if limit:
            num_datapoints = min(int(limit), num_datapoints)

        return int(math.ceil(num_datapoints))


Instruction:
How would you implement a function in Python 3 that
creates a series with num_series datapoints and write them to the client.

Response:
def create_series(self, num_series, batch_size=5000):
        """
        Write one data point for each series name to initialize the series
        :param num_series: Number of different series names to create
        :param batch_size: Number of series to create at the same time
        :return:
        """
        datapoints = []
        for _ in range(num_series):
            name = self.dummy_seriesname()
            datapoints.append(self.create_datapoint(name, ["value"], [[1]]))
        for data in tqdm(self.batch(datapoints, batch_size)):
            self.client.write_points(data)


Instruction:
Make a summary of the following Python 3 code
def write_points(self, series_name, start_date, end_date, resolution=10, batch_size=5000):
        start_ts = int(start_date.strftime("%s"))
        end_ts = int(end_date.strftime("%s"))

        range_seconds = end_ts - start_ts
        num_datapoints = range_seconds / resolution

        timestamps = [start_ts + i * resolution for i in range(num_datapoints)]

        columns = ["time", "value"]
        for batch in tqdm(self.batch(timestamps, batch_size)):
            points = []
            for timestamp in batch:
                point = random.randint(1, 100)
                points.append([timestamp, point])
            datapoint = self.create_datapoint(series_name, columns, points)
            self.client.write_points([datapoint])

Response: Create sample datapoints between two dates with the given resolution


Instruction:
Can you tell what is the following Python 3 function doing
def check(self, query):
        if query.get_type() not in {Keyword.SELECT}:
            # Bailing out for non select queries
            return Ok(True)

        if query.get_resolution() > 0:
            return Ok(True)

        return Err("Group by statements need a positive time value (e.g. time(10s))")

Response: Check that the query is valid.


Instruction:
Can you implement a function in Python 3 that
parses a raw InfluxDB query string into a dictionary of fields.

Response:
def parse(self, raw_query_string):
        """
        Parse a raw query string into fields
        :param raw_query_string: Raw InfluxDB query string
        """

        self._reset()

        if not isinstance(raw_query_string, basestring):
            return None

        query_string = self._cleanup(raw_query_string)
        parts = self._split(query_string)
        parts = self._sanitize_keywords(parts)
        tokens = self._tokenize(parts)

        if tokens:
            # Run subparsers to analyze parts of the query
            self.parsed_resolution = self._parse_resolution(tokens)
            self.parsed_time = self._parse_time(tokens)
            self.parsed_time_overlap = self._parse_duration(self.parsed_time)
            self.parsed_datapoints = self._parse_datapoints(
                self.parsed_time_overlap.timespan_seconds(),
                self.parsed_resolution,
                self.parse_keyword(Keyword.LIMIT, tokens)
            )

        return self.create_query_object(tokens)


Instruction:
Write a Python 3 function for
analyzing query tokens and create an InfluxDBStatement from them

Response:
def create_query_object(self, tokens):
        """
        Analyze query tokens and create an InfluxDBStatement from them
        Return None on error
        :param tokens: A list of InfluxDB query tokens
        """
        try:
            query_type = tokens['type']
            return getattr(self, 'create_%s_query' % query_type)(tokens)
        except (KeyError, TypeError):
            return self.invalid_query(tokens)


Instruction:
Make a summary of the following Python 3 code
def create_select_query(self, tokens):
        if not tokens[Keyword.SELECT]:
            return None
        if not tokens[Keyword.FROM]:
            return None

        return SelectQuery(
            self.parse_keyword(Keyword.SELECT, tokens),
            self.parse_keyword(Keyword.FROM, tokens),
            where_stmt=self.parse_keyword(Keyword.WHERE, tokens),
            limit_stmt=self.parse_keyword(Keyword.LIMIT, tokens),
            group_by_stmt=self.parse_group(tokens),
            duration=self.parsed_time_overlap.timespan_seconds(),
            resolution=self.parsed_resolution,
            time_ranges=self.parsed_time,
            time_overlap=self.parsed_time_overlap,
            datapoints=self.parsed_datapoints
        )

Response: Parse the list of tokens of select query


Instruction:
Can you generate the documentation for the following Python 3 function
def create_list_query(self, tokens):
        if not tokens[Keyword.SERIES]:
            # A list series keyword is allowed
            # without a series name or regex
            tokens[Keyword.SERIES] = ''
        return ListQuery(self.parse_keyword(Keyword.SERIES, tokens))

Response: Parse the list query from the tokens.


Instruction:
How would you implement a function in Python 3 that
parses the list of tokens of drop query

Response:
def create_drop_query(self, tokens):
        """
        Parse tokens of drop query
        :param tokens: A list of InfluxDB query tokens
        """
        if not tokens[Keyword.SERIES]:
            return None
        return DropQuery(self.parse_keyword(Keyword.SERIES, tokens))


Instruction:
Explain what the following Python 3 code does
def create_delete_query(self, tokens):
        # From keyword is required
        if not tokens[Keyword.FROM]:
            return None
        where_stmt = self.parse_keyword(Keyword.WHERE, tokens)
        if where_stmt:
            if not where_stmt.startswith('time'):
                return None
        return DeleteQuery(
            self.parse_keyword(Keyword.FROM, tokens),
            self.parse_keyword(Keyword.WHERE, tokens)
        )

Response: Parse the tokens of delete query


Instruction:
Here you have a function in Python 3, explain what it does
def _parse_time(self, tokens):
        return self.time_parser.parse(self.parse_keyword(Keyword.WHERE, tokens))

Response:
Parse the time range for the query
        Eg. WHERE time < now - 48h and time < now - 24h


Instruction:
Implement a function in Python 3 to
parse the resolution from the GROUP BY statement.

Response:
def _parse_resolution(self, tokens):
        """
        Parse resolution from the GROUP BY statement.
        E.g. GROUP BY time(10s) would mean a 10 second resolution
        :param tokens:
        :return:
        """
        return self.resolution_parser.parse(self.parse_keyword(Keyword.GROUP_BY, tokens))


Instruction:
Implement a function in Python 3 to
parse the number of datapoints of a query.

Response:
def _parse_datapoints(self, parsed_duration, parsed_resolution, limit):
        """
        Parse the number of datapoints of a query.
        This can be calculated from the given duration and resolution of the query.
        E.g. if the query has a duation of 2*60*60 = 7200 seconds and a resolution of 10 seconds
        then the number of datapoints would be 7200/10 => 7200 datapoints.

        :param parsed_duration:
        :param parsed_resolution:
        :param limit:
        :return:
        """
        return self.datapoints_parser.parse(parsed_duration, parsed_resolution, limit)


Instruction:
How would you implement a function in Python 3 that
checks if the current time is before the earliest date.

Response:
def check(self, query):
        """
        :param query:
        """
        if query.get_type() not in {Keyword.SELECT}:
            # Only select queries need to be checked here
            # All others are not affected by this rule. Bailing out.
            return Ok(True)

        earliest_date = query.get_earliest_date()
        if earliest_date >= self.min_start_date:
            return Ok(True)

        if query.limit_stmt:
            return Ok(True)

        return Err(("Querying for data before {} is prohibited. "
                    "Your beginning date is {}, which is before that.").format(self.min_start_date.strftime("%Y-%m-%d"),
                                                                              earliest_date))


Instruction:
Can you create a Python 3 function that
extracts date from string if necessary.

Response:
def extract_date(self, date):
        """Extract date from string if necessary.

        :returns: the extracted date.
        """
        if isinstance(date, six.string_types):
            try:
                date = dateutil.parser.parse(date)
            except ValueError:
                raise ValueError(
                    'Invalid date format for statistic {}.'
                ).format(self.query_name)
        if not isinstance(date, datetime):
            raise TypeError(
                'Invalid date type for statistic {}.'
            ).format(self.query_name)
        return date


Instruction:
Given the following Python 3 function, write the documentation
def validate_arguments(self, interval, start_date, end_date, **kwargs):
        if interval not in self.allowed_intervals:
            raise InvalidRequestInputError(
                'Invalid aggregation time interval for statistic {}.'
            ).format(self.query_name)
        if set(kwargs) < set(self.required_filters):
            raise InvalidRequestInputError(
                'Missing one of the required parameters {0} in '
                'query {1}'.format(set(self.required_filters.keys()),
                                   self.query_name)
            )

Response: Validate the query arguments.


Instruction:
Can you generate the documentation for the following Python 3 function
def build_query(self, interval, start_date, end_date, **kwargs):
        agg_query = Search(using=self.client,
                           index=self.index,
                           doc_type=self.doc_type)[0:0]
        if start_date is not None or end_date is not None:
            time_range = {}
            if start_date is not None:
                time_range['gte'] = start_date.isoformat()
            if end_date is not None:
                time_range['lte'] = end_date.isoformat()
            agg_query = agg_query.filter(
                'range',
                **{self.time_field: time_range})

        for modifier in self.query_modifiers:
            agg_query = modifier(agg_query, **kwargs)

        base_agg = agg_query.aggs.bucket(
            'histogram',
            'date_histogram',
            field=self.time_field,
            interval=interval
        )

        for destination, (metric, field, opts) in self.metric_fields.items():
            base_agg.metric(destination, metric, field=field, **opts)

        if self.copy_fields:
            base_agg.metric(
                'top_hit', 'top_hits', size=1, sort={'timestamp': 'desc'}
            )

        for query_param, filtered_field in self.required_filters.items():
            if query_param in kwargs:
                agg_query = agg_query.filter(
                    'term', **{filtered_field: kwargs[query_param]}
                )

        return agg_query

Response: Build the elasticsearch query.


Instruction:
Can you generate the documentation for the following Python 3 function
def process_query_result(self, query_result, interval,
                             start_date, end_date):
        """Build the result using the query result."""
        def build_buckets(agg):
            """Build recursively result buckets."""
            bucket_result = dict(
                key=agg['key'],
                date=agg['key_as_string'],
            )
            for metric in self.metric_fields:
                bucket_result[metric] = agg[metric]['value']
            if self.copy_fields and agg['top_hit']['hits']['hits']:
                doc = agg['top_hit']['hits']['hits'][0]['_source']
                for destination, source in self.copy_fields.items():
                    if isinstance(source, six.string_types):
                        bucket_result[destination] = doc[source]
                    else:
                        bucket_result[destination] = source(bucket_result, doc)
            return bucket_result

        # Add copy_fields
        buckets = query_result['aggregations']['histogram']['buckets']
        return dict(
            interval=interval,
            key_type='date',
            start_date=start_date.isoformat() if start_date else None,
            end_date=end_date.isoformat() if end_date else None,
            buckets=[build_buckets(b) for b in buckets]
        )

Response: Process the query result.


Instruction:
Create a Python 3 function for
validating the arguments for the query.

Response:
def validate_arguments(self, start_date, end_date, **kwargs):
        """Validate query arguments."""
        if set(kwargs) < set(self.required_filters):
            raise InvalidRequestInputError(
                'Missing one of the required parameters {0} in '
                'query {1}'.format(set(self.required_filters.keys()),
                                   self.query_name)
            )


Instruction:
Here you have a function in Python 3, explain what it does
def build_query(self, start_date, end_date, **kwargs):
        agg_query = Search(using=self.client,
                           index=self.index,
                           doc_type=self.doc_type)[0:0]
        if start_date is not None or end_date is not None:
            time_range = {}
            if start_date is not None:
                time_range['gte'] = start_date.isoformat()
            if end_date is not None:
                time_range['lte'] = end_date.isoformat()
            agg_query = agg_query.filter(
                'range',
                **{self.time_field: time_range})

        for modifier in self.query_modifiers:
            agg_query = modifier(agg_query, **kwargs)

        base_agg = agg_query.aggs

        def _apply_metric_aggs(agg):
            for dst, (metric, field, opts) in self.metric_fields.items():
                agg.metric(dst, metric, field=field, **opts)

        _apply_metric_aggs(base_agg)
        if self.aggregated_fields:
            cur_agg = base_agg
            for term in self.aggregated_fields:
                cur_agg = cur_agg.bucket(term, 'terms', field=term, size=0)
                _apply_metric_aggs(cur_agg)

        if self.copy_fields:
            base_agg.metric(
                'top_hit', 'top_hits', size=1, sort={'timestamp': 'desc'}
            )

        for query_param, filtered_field in self.required_filters.items():
            if query_param in kwargs:
                agg_query = agg_query.filter(
                    'term', **{filtered_field: kwargs[query_param]}
                )

        return agg_query

Response: Build the elasticsearch query.


Instruction:
Can you create a Python 3 function that
processes the query result and return a dictionary of the result.

Response:
def process_query_result(self, query_result, start_date, end_date):
        """Build the result using the query result."""
        def build_buckets(agg, fields, bucket_result):
            """Build recursively result buckets."""
            # Add metric results for current bucket
            for metric in self.metric_fields:
                bucket_result[metric] = agg[metric]['value']
            if fields:
                current_level = fields[0]
                bucket_result.update(dict(
                    type='bucket',
                    field=current_level,
                    key_type='terms',
                    buckets=[build_buckets(b, fields[1:], dict(key=b['key']))
                             for b in agg[current_level]['buckets']]
                ))
            return bucket_result

        # Add copy_fields
        aggs = query_result['aggregations']
        result = dict(
            start_date=start_date.isoformat() if start_date else None,
            end_date=end_date.isoformat() if end_date else None,
        )
        if self.copy_fields and aggs['top_hit']['hits']['hits']:
            doc = aggs['top_hit']['hits']['hits'][0]['_source']
            for destination, source in self.copy_fields.items():
                if isinstance(source, six.string_types):
                    result[destination] = doc[source]
                else:
                    result[destination] = source(result, doc)

        return build_buckets(aggs, self.aggregated_fields, result)


Instruction:
Create a Python 3 function to
override error handling to suppress socket and ssl related errors

Response:
def handle_error(self, request, client_address):
        """
        Overwrite error handling to suppress socket/ssl related errors
        :param client_address: Address of client
        :param request: Request causing an error
        """
        cls, e = sys.exc_info()[:2]
        if cls is socket.error or cls is ssl.SSLError:
            pass
        else:
            return HTTPServer.handle_error(self, request, client_address)


Instruction:
Write a Python 3 function for
building a file - download event.

Response:
def file_download_event_builder(event, sender_app, obj=None, **kwargs):
    """Build a file-download event."""
    event.update(dict(
        # When:
        timestamp=datetime.datetime.utcnow().isoformat(),
        # What:
        bucket_id=str(obj.bucket_id),
        file_id=str(obj.file_id),
        file_key=obj.key,
        size=obj.file.size,
        referrer=request.referrer,
        # Who:
        **get_user()
    ))
    return event


Instruction:
Implement a function in Python 3 to
build a record - view event.

Response:
def record_view_event_builder(event, sender_app, pid=None, record=None,
                              **kwargs):
    """Build a record-view event."""
    event.update(dict(
        # When:
        timestamp=datetime.datetime.utcnow().isoformat(),
        # What:
        record_id=str(record.id),
        pid_type=pid.pid_type,
        pid_value=str(pid.pid_value),
        referrer=request.referrer,
        # Who:
        **get_user()
    ))
    return event


Instruction:
Can you generate the documentation for the following Python 3 function
def check_write_permissions(file):
    try:
        open(file, 'a')
    except IOError:
        print("Can't open file {}. "
              "Please grant write permissions or change the path in your config".format(file))
        sys.exit(1)

Response: Check if we can write to the given file


Instruction:
How would you explain what the following Python 3 function does
def show_rules():
    from rules.loader import import_rules
    from rules.rule_list import all_rules
    rules = import_rules(all_rules)
    print("")
    for name, rule in rules.iteritems():
        heading = "{} (`{}`)".format(rule.description(), name)
        print("#### {} ####".format(heading))
        for line in rule.reason():
            print(line)
        print("")
    sys.exit(0)

Response: Show the list of available rules and quit


Instruction:
How would you implement a function in Python 3 that
starts the http proxy

Response:
def start_proxy(config):
    """
    Start the http proxy
    :param config:
    :return:
    """
    protector = Protector(config.rules, config.whitelist)
    protector_daemon = ProtectorDaemon(config=config, protector=protector)

    daemon = daemonocle.Daemon(
        pidfile=config.pidfile,
        detach=(not config.foreground),
        shutdown_callback=shutdown,
        worker=protector_daemon.run
    )
    daemon.do_action(config.command)


Instruction:
Write a Python 3 function for
yielding n batches of the sequence.

Response:
def batches(iterable, n=1):
    """
    From http://stackoverflow.com/a/8290508/270334
    :param n:
    :param iterable:
    """
    l = len(iterable)
    for ndx in range(0, l, n):
        yield iterable[ndx:min(ndx + n, l)]


Instruction:
Can you generate the documentation for the following Python 3 function
def _is_root():
    import os
    import ctypes
    try:
        return os.geteuid() == 0
    except AttributeError:
        return ctypes.windll.shell32.IsUserAnAdmin() != 0
    return False

Response: Checks if the user is rooted.


Instruction:
Given the following Python 3 function, write the documentation
def cmdclass(path, enable=None, user=None):
    import warnings
    from setuptools.command.install import install
    from setuptools.command.develop import develop
    from os.path import dirname, join, exists, realpath
    from traceback import extract_stack

    try:
        # IPython/Jupyter 4.0
        from notebook.nbextensions import install_nbextension
        from notebook.services.config import ConfigManager
    except ImportError:
        # Pre-schism
        try:
            from IPython.html.nbextensions import install_nbextension
            from IPython.html.services.config import ConfigManager
        except ImportError:
            warnings.warn("No jupyter notebook found in your environment. "
                          "Hence jupyter nbextensions were not installed. "
                          "If you would like to have them,"
                          "please issue 'pip install jupyter'.")
            return {}

    # Check if the user flag was set.
    if user is None:
        user = not _is_root()

    # Get the path of the extension
    calling_file = extract_stack()[-2][0]
    fullpath = realpath(calling_file)
    if not exists(fullpath):
        raise Exception('Could not find path of setup file.')
    extension_dir = join(dirname(fullpath), path)

    # Installs the nbextension
    def run_nbextension_install(develop):
        import sys
        sysprefix = hasattr(sys, 'real_prefix')
        if sysprefix:
            install_nbextension(
                extension_dir, symlink=develop, sys_prefix=sysprefix)
        else:
            install_nbextension(extension_dir, symlink=develop, user=user)
        if enable is not None:
            print("Enabling the extension ...")
            cm = ConfigManager()
            cm.update('notebook', {"load_extensions": {enable: True}})

    # Command used for standard installs
    class InstallCommand(install):
        def run(self):
            print("Installing Python module...")
            install.run(self)
            print("Installing nbextension ...")
            run_nbextension_install(False)

    # Command used for development installs (symlinks the JS)
    class DevelopCommand(develop):
        def run(self):
            print("Installing Python module...")
            develop.run(self)
            print("Installing nbextension ...")
            run_nbextension_install(True)

    return {
        'install': InstallCommand,
        'develop': DevelopCommand,
    }

Response: Build the nbextension commandclass dict for the notebook notebook.


Instruction:
Can you tell what is the following Python 3 function doing
def count_subgraph_sizes(graph: BELGraph, annotation: str = 'Subgraph') -> Counter[int]:
    return count_dict_values(group_nodes_by_annotation(graph, annotation))

Response: Count the number of nodes in each subgraph induced by an annotation.


Instruction:
Here you have a function in Python 3, explain what it does
def calculate_subgraph_edge_overlap(
        graph: BELGraph,
        annotation: str = 'Subgraph'
) -> Tuple[
    Mapping[str, EdgeSet],
    Mapping[str, Mapping[str, EdgeSet]],
    Mapping[str, Mapping[str, EdgeSet]],
    Mapping[str, Mapping[str, float]],
]:
    """Build a DatafFame to show the overlap between different sub-graphs.

    Options:
    1. Total number of edges overlap (intersection)
    2. Percentage overlap (tanimoto similarity)

    :param graph: A BEL graph
    :param annotation: The annotation to group by and compare. Defaults to 'Subgraph'
    :return: {subgraph: set of edges}, {(subgraph 1, subgraph2): set of intersecting edges},
            {(subgraph 1, subgraph2): set of unioned edges}, {(subgraph 1, subgraph2): tanimoto similarity},
    """
    sg2edge = defaultdict(set)

    for u, v, d in graph.edges(data=True):
        if not edge_has_annotation(d, annotation):
            continue
        sg2edge[d[ANNOTATIONS][annotation]].add((u, v))

    subgraph_intersection = defaultdict(dict)
    subgraph_union = defaultdict(dict)
    result = defaultdict(dict)

    for sg1, sg2 in itt.product(sg2edge, repeat=2):
        subgraph_intersection[sg1][sg2] = sg2edge[sg1] & sg2edge[sg2]
        subgraph_union[sg1][sg2] = sg2edge[sg1] | sg2edge[sg2]
        result[sg1][sg2] = len(subgraph_intersection[sg1][sg2]) / len(subgraph_union[sg1][sg2])

    return sg2edge, subgraph_intersection, subgraph_union, result

Response: Calculate the overlap between two sub - graphs.


Instruction:
Make a summary of the following Python 3 code
def summarize_subgraph_edge_overlap(graph: BELGraph, annotation: str = 'Subgraph') -> Mapping[str, Mapping[str, float]]:
    _, _, _, subgraph_overlap = calculate_subgraph_edge_overlap(graph, annotation)
    return subgraph_overlap

Response: Summarize the similarity matrix between all subgraphs or other given annotation.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def summarize_subgraph_node_overlap(graph: BELGraph, node_predicates=None, annotation: str = 'Subgraph'):
    r1 = group_nodes_by_annotation_filtered(graph, node_predicates=node_predicates, annotation=annotation)
    return calculate_tanimoto_set_distances(r1)

Response: Summarize the similarity between nodes passing the given filter.


Instruction:
Explain what the following Python 3 code does
def rank_subgraph_by_node_filter(graph: BELGraph,
                                 node_predicates: Union[NodePredicate, Iterable[NodePredicate]],
                                 annotation: str = 'Subgraph',
                                 reverse: bool = True,
                                 ) -> List[Tuple[str, int]]:
    """Rank sub-graphs by which have the most nodes matching an given filter.

    A use case for this function would be to identify which subgraphs contain the most differentially expressed
    genes.

    >>> from pybel import from_pickle
    >>> from pybel.constants import GENE
    >>> from pybel_tools.integration import overlay_type_data
    >>> from pybel_tools.summary import rank_subgraph_by_node_filter
    >>> import pandas as pd
    >>> graph = from_pickle('~/dev/bms/aetionomy/alzheimers.gpickle')
    >>> df = pd.read_csv('~/dev/bananas/data/alzheimers_dgxp.csv', columns=['Gene', 'log2fc'])
    >>> data = {gene: log2fc for _, gene, log2fc in df.itertuples()}
    >>> overlay_type_data(graph, data, 'log2fc', GENE, 'HGNC', impute=0.0)
    >>> results = rank_subgraph_by_node_filter(graph, lambda g, n: 1.3 < abs(g[n]['log2fc']))
    """
    r1 = group_nodes_by_annotation_filtered(graph, node_predicates=node_predicates, annotation=annotation)
    r2 = count_dict_values(r1)
    # TODO use instead: r2.most_common()
    return sorted(r2.items(), key=itemgetter(1), reverse=reverse)

Response: Rank subgraphs by which have the most nodes matching a given filter.


Instruction:
Create a Python 3 function for
rendering the graph as JavaScript in a Jupyter Notebook.

Response:
def to_jupyter(graph: BELGraph, chart: Optional[str] = None) -> Javascript:
    """Render the graph as JavaScript in a Jupyter Notebook."""
    with open(os.path.join(HERE, 'render_with_javascript.js'), 'rt') as f:
        js_template = Template(f.read())

    return Javascript(js_template.render(**_get_context(graph, chart=chart)))


Instruction:
Can you tell what is the following Python 3 function doing
def to_html(graph: BELGraph, chart: Optional[str] = None) -> str:
    with open(os.path.join(HERE, 'index.html'), 'rt') as f:
        html_template = Template(f.read())

    return html_template.render(**_get_context(graph, chart=chart))

Response: Render the graph as an HTML string.


Instruction:
How would you explain what the following Python 3 function does
def prerender(graph: BELGraph) -> Mapping[str, Mapping[str, Any]]:
    import bio2bel_hgnc
    from bio2bel_hgnc.models import HumanGene

    graph: BELGraph = graph.copy()
    enrich_protein_and_rna_origins(graph)
    collapse_all_variants(graph)
    genes: Set[Gene] = get_nodes_by_function(graph, GENE)
    hgnc_symbols = {
        gene.name
        for gene in genes
        if gene.namespace.lower() == 'hgnc'
    }

    result = {}

    hgnc_manager = bio2bel_hgnc.Manager()
    human_genes = (
        hgnc_manager.session
            .query(HumanGene.symbol, HumanGene.location)
            .filter(HumanGene.symbol.in_(hgnc_symbols))
            .all()
    )
    for human_gene in human_genes:
        result[human_gene.symbol] = {
            'name': human_gene.symbol,
            'chr': (
                human_gene.location.split('q')[0]
                if 'q' in human_gene.location else
                human_gene.location.split('p')[0]
            ),
        }

    df = get_df()

    for _, (gene_id, symbol, start, stop) in df[df['Symbol'].isin(hgnc_symbols)].iterrows():
        result[symbol]['start'] = start
        result[symbol]['stop'] = stop

    return result

Response: Generate the annotations JSON for Ideogram.


Instruction:
Can you write a function in Python 3 where it
plots your graph summary statistics on the given axes.

Response:
def plot_summary_axes(graph: BELGraph, lax, rax, logx=True):
    """Plots your graph summary statistics on the given axes.

    After, you should run :func:`plt.tight_layout` and you must run :func:`plt.show` to view.

    Shows:
    1. Count of nodes, grouped by function type
    2. Count of edges, grouped by relation type

    :param pybel.BELGraph graph: A BEL graph
    :param lax: An axis object from matplotlib
    :param rax: An axis object from matplotlib

    Example usage:

    >>> import matplotlib.pyplot as plt
    >>> from pybel import from_pickle
    >>> from pybel_tools.summary import plot_summary_axes
    >>> graph = from_pickle('~/dev/bms/aetionomy/parkinsons.gpickle')
    >>> fig, axes = plt.subplots(1, 2, figsize=(10, 4))
    >>> plot_summary_axes(graph, axes[0], axes[1])
    >>> plt.tight_layout()
    >>> plt.show()
    """
    ntc = count_functions(graph)
    etc = count_relations(graph)

    df = pd.DataFrame.from_dict(dict(ntc), orient='index')
    df_ec = pd.DataFrame.from_dict(dict(etc), orient='index')

    df.sort_values(0, ascending=True).plot(kind='barh', logx=logx, ax=lax)
    lax.set_title('Number of nodes: {}'.format(graph.number_of_nodes()))

    df_ec.sort_values(0, ascending=True).plot(kind='barh', logx=logx, ax=rax)
    rax.set_title('Number of edges: {}'.format(graph.number_of_edges()))


Instruction:
Here you have a function in Python 3, explain what it does
def plot_summary(graph: BELGraph, plt, logx=True, **kwargs):
    fig, axes = plt.subplots(1, 2, **kwargs)
    lax = axes[0]
    rax = axes[1]

    plot_summary_axes(graph, lax, rax, logx=logx)
    plt.tight_layout()

    return fig, axes

Response: Plots your graph summary statistics.