Instruction:
Explain what the following Python 3 code does
def normalize_type(self, names):
        self.name = names[0]
        self.is_kwarg = False
        self.is_arg = False
        self.names = []

        try:
            # http://stackoverflow.com/a/16488383/5006 uses ask forgiveness because
            # of py2/3 differences of integer check
            self.index = int(self.name)
            self.name = ""
            self.is_arg = True

        except ValueError:
            self.is_kwarg = True
            self.names = names

Response: Decide if this param is an arg or a kwarg and set appropriate internal flags


Instruction:
Can you generate the documentation for the following Python 3 function
def normalize_param(self, slf, args, kwargs):
        if self.is_kwarg:
            kwargs = self.normalize_kwarg(slf.request, kwargs)
        else:
            args = self.normalize_arg(slf.request, args)
        return slf, args, kwargs

Response:
this function will try to find the param and
            put it in args if it has a default and stuff


Instruction:
Explain what the following Python 3 code does
def find_kwarg(self, request, names, required, default, kwargs):
        val = default
        found_name = ''
        for name in names:
            if name in kwargs:
                val = kwargs[name]
                found_name = name
                break

        if not found_name and required:
            raise ValueError("required param {} does not exist".format(self.name))

        return found_name, val

Response: actually try to retrieve names key from params dict


Instruction:
Make a summary of the following Python 3 code
def normalize_val(self, request, val):
        flags = self.flags
        paction = flags['action']
        ptype = flags['type']
        pchoices = flags.get('choices', None)
        allow_empty = flags.get('allow_empty', False)
        min_size = flags.get('min_size', None)
        max_size = flags.get('max_size', None)
        regex = flags.get('regex', None)

        if paction in set(['store_list']):
            if isinstance(val, list) and len(val) > 1:
                raise ValueError("too many values for param")

            if isinstance(val, basestring):
                val = list(val.split(','))

            else:
                val = list(val)

        elif paction in set(['append', 'append_list']):
            if not isinstance(val, list):
                val = [val]

            if paction == 'append_list':
                vs = []
                for v in val:
                    if isinstance(v, basestring):
                        vs.extend(v.split(','))
                    else:
                        vs.append(v)

                val = vs

        else:
            if paction not in set(['store', 'store_false', 'store_true']):
                raise RuntimeError('unknown param action {}'.format(paction))

        if regex:
            failed = False
            if isinstance(regex, basestring):
                if not re.search(regex, val): failed = True
            else:
                if not regex.search(val): failed = True

            if failed:
                raise ValueError("param failed regex check")

        if ptype:
            if isinstance(val, list) and ptype != list:
                val = list(map(ptype, val))

            else:
                if isinstance(ptype, type):
                    if issubclass(ptype, bool):
                        if val in set(['true', 'True', '1']):
                            val = True
                        elif val in set(['false', 'False', '0']):
                            val = False
                        else:
                            val = ptype(val)

                    elif issubclass(ptype, str):
                        charset = request.encoding
                        if is_py2:
                            val = ptype(ByteString(val, charset))
                        else:
                            val = ptype(String(val, charset))

#                         if charset and isinstance(val, unicode):
#                             val = val.encode(charset)
#                         else:
#                             val = ptype(val)

                    else:
                        val = ptype(val)

                else:
                    val = ptype(val)

        if pchoices:
            if isinstance(val, list) and ptype != list:
                for v in val:
                    if v not in pchoices:
                        raise ValueError("param value {} not in choices {}".format(v, pchoices))

            else:
                if val not in pchoices:
                    raise ValueError("param value {} not in choices {}".format(val, pchoices))

        # at some point this if statement is just going to be too ridiculous
        # FieldStorage check is because of this bug https://bugs.python.org/issue19097
        if not isinstance(val, cgi.FieldStorage):
            if not allow_empty and val is not False and not val:
                if 'default' not in flags:
                    raise ValueError("param was empty")

        if min_size is not None:
            failed = False
            if isinstance(val, (int, float)):
                if val < min_size: failed = True
            else:
                if len(val) < min_size: failed = True

            if failed:
                raise ValueError("param was smaller than {}".format(min_size))

        if max_size is not None:
            failed = False
            if isinstance(val, (int, float)):
                if val > max_size: failed = True
            else:
                if len(val) > max_size: failed = True

            if failed:
                raise ValueError("param was bigger than {}".format(max_size))

        return val

Response: This will take the value and make sure it meets expectations


Instruction:
Make a summary of the following Python 3 code
def connect(self, path="", headers=None, query=None, timeout=0, **kwargs):
        ret = None
        ws_url = self.get_fetch_url(path, query)
        ws_headers = self.get_fetch_headers("GET", headers)
        ws_headers = ['{}: {}'.format(h[0], h[1]) for h in ws_headers.items() if h[1]]
        timeout = self.get_timeout(timeout=timeout, **kwargs)

        self.set_trace(kwargs.pop("trace", False))
        #pout.v(websocket_url, websocket_headers, self.query_kwargs, self.headers)

        try:
            logger.debug("{} connecting to {}".format(self.client_id, ws_url))
            self.ws = websocket.create_connection(
                ws_url,
                header=ws_headers,
                timeout=timeout,
                sslopt={'cert_reqs':ssl.CERT_NONE},
            )

            ret = self.recv_callback(callback=lambda r: r.uuid == "CONNECT")
            if ret.code >= 400:
                raise IOError("Failed to connect with code {}".format(ret.code))

#             self.headers = headers
#             self.query_kwargs = query_kwargs

        except websocket.WebSocketTimeoutException:
            raise IOError("Failed to connect within {} seconds".format(timeout))

        except websocket.WebSocketException as e:
            raise IOError("Failed to connect with error: {}".format(e))

        except socket.error as e:
            # this is an IOError, I just wanted to be aware of that, most common
            # problem is: [Errno 111] Connection refused
            raise

        return ret

Response: Connect to the most common ACS - related API.


Instruction:
Can you tell what is the following Python 3 function doing
def fetch(self, method, path, query=None, body=None, timeout=0, **kwargs):
        ret = None
        if not query: query = {}
        if not body: body = {}
        query.update(body) # body takes precedence
        body = query

        self.send_count += 1
        payload = self.get_fetch_request(method, path, body)
        attempts = 1
        max_attempts = self.attempts
        success = False

        while not success:
            kwargs['timeout'] = timeout
            try:
                try:
                    if not self.connected: self.connect(path)
                    with self.wstimeout(**kwargs) as timeout:
                        kwargs['timeout'] = timeout

                        logger.debug('{} send {} attempt {}/{} with timeout {}'.format(
                            self.client_id,
                            payload.uuid,
                            attempts,
                            max_attempts,
                            timeout
                        ))

                        sent_bits = self.ws.send(payload.payload)
                        logger.debug('{} sent {} bytes'.format(self.client_id, sent_bits))
                        if sent_bits:
                            ret = self.fetch_response(payload, **kwargs)
                            if ret:
                                success = True

                except websocket.WebSocketConnectionClosedException as e:
                    self.ws.shutdown()
                    raise IOError("connection is not open but reported it was open: {}".format(e))

            except (IOError, TypeError) as e:
                logger.debug('{} error on send attempt {}: {}'.format(self.client_id, attempts, e))
                success = False

            finally:
                if not success:
                    attempts += 1
                    if attempts > max_attempts:
                        raise

                    else:
                        timeout *= 2
                        if (attempts / max_attempts) > 0.50:
                            logger.debug(
                                "{} closing and re-opening connection for next attempt".format(self.client_id)
                            )
                            self.close()

        return ret

Response: send a message to the server


Instruction:
How would you explain what the following Python 3 function does
def fetch_response(self, req_payload, **kwargs):
        if req_payload.uuid:
            uuids = set([req_payload.uuid, "CONNECT"])
            def callback(res_payload):
                #pout.v(req_payload, res_payload)
                #ret = req_payload.uuid == res_payload.uuid or res_payload.uuid == "CONNECT"
                ret = res_payload.uuid in uuids
                if ret:
                    logger.debug('{} received {} response for {}'.format(
                        self.client_id,
                        res_payload.code,
                        res_payload.uuid,
                    ))
                return ret

            res_payload = self.recv_callback(callback, **kwargs)

        return res_payload

Response: fetch_response - send a response to the server


Instruction:
How would you code a function in Python 3 to
ping the server and check the status of the user s agent.

Response:
def ping(self, timeout=0, **kwargs):
        """THIS DOES NOT WORK, UWSGI DOES NOT RESPOND TO PINGS"""

        # http://stackoverflow.com/a/2257449/5006
        def rand_id(size=8, chars=string.ascii_uppercase + string.digits):
            return ''.join(random.choice(chars) for _ in range(size))

        payload = rand_id()
        self.ws.ping(payload)
        opcode, data = self.recv_raw(timeout, [websocket.ABNF.OPCODE_PONG], **kwargs)
        if data != payload:
            raise IOError("Pinged server but did not receive correct pong")


Instruction:
Implement a Python 3 function for
receiving messages and validate them with the callback

Response:
def recv_callback(self, callback, **kwargs):
        """receive messages and validate them with the callback, if the callback
        returns True then the message is valid and will be returned, if False then
        this will try and receive another message until timeout is 0"""
        payload = None
        timeout = self.get_timeout(**kwargs)
        full_timeout = timeout
        while timeout > 0.0:
            kwargs['timeout'] = timeout
            start = time.time()
            payload = self.recv(**kwargs)
            if callback(payload):
                break
            payload = None
            stop = time.time()
            elapsed = stop - start
            timeout -= elapsed

        if not payload:
            raise IOError("recv_callback timed out in {}".format(full_timeout))

        return payload


Instruction:
Can you write a function in Python 3 where it
creates a new controller instance for the request and response.

Response:
def create_controller(self):
        """Create a controller to handle the request

        :returns: Controller, this Controller instance should be able to handle
            the request
        """
        body = None
        req = self.request
        res = self.response
        rou = self.router
        con = None

        controller_info = {}
        try:
            controller_info = rou.find(req, res)

        except IOError as e:
            logger.warning(str(e), exc_info=True)
            raise CallError(
                408,
                "The client went away before the request body was retrieved."
            )

        except (ImportError, AttributeError, TypeError) as e:
            exc_info = sys.exc_info()
            logger.warning(str(e), exc_info=exc_info)
            raise CallError(
                404,
                "{} not found because of {} \"{}\" on {}:{}".format(
                    req.path,
                    exc_info[0].__name__,
                    str(e),
                    os.path.basename(exc_info[2].tb_frame.f_code.co_filename),
                    exc_info[2].tb_lineno
                )
            )

        else:
            con = controller_info['class_instance']

        return con


Instruction:
Explain what the following Python 3 code does
def handle(self):
        body = None
        req = self.request
        res = self.response
        rou = self.router
        con = None
        start = time.time()

        try:
            con = self.create_controller()
            con.call = self
            self.controller = con
            if not self.quiet:
                con.log_start(start)

            # the controller handle method will manipulate self.response, it first
            # tries to find a handle_HTTP_METHOD method, if it can't find that it
            # will default to the handle method (which is implemented on Controller).
            # method arguments are passed in so child classes can add decorators
            # just like the HTTP_METHOD that will actually handle the request
            controller_args, controller_kwargs = con.find_method_params()
            controller_method = getattr(con, "handle_{}".format(req.method), None)
            if not controller_method:
                controller_method = getattr(con, "handle")

            if not self.quiet:
                logger.debug("Using handle method: {}.{}".format(
                    con.__class__.__name__,
                    controller_method.__name__
                ))
            controller_method(*controller_args, **controller_kwargs)

        except Exception as e:
            self.handle_error(e) # this will manipulate self.response

        finally:
            if res.code == 204:
                res.headers.pop('Content-Type', None)
                res.body = None # just to be sure since body could've been ""

            if con:
                if not self.quiet:
                    con.log_stop(start)

        return res

Response: Called from the interface to actually handle the request.


Instruction:
Can you tell what is the following Python 3 function doing
def handle_error(self, e, **kwargs):
        req = self.request
        res = self.response
        con = self.controller

        if isinstance(e, CallStop):
            logger.info(str(e), exc_info=True)
            res.code = e.code
            res.add_headers(e.headers)
            res.body = e.body

        elif isinstance(e, Redirect):
            logger.info(str(e), exc_info=True)
            res.code = e.code
            res.add_headers(e.headers)
            res.body = None

        elif isinstance(e, (AccessDenied, CallError)):
            logger.warning(str(e), exc_info=True)
            res.code = e.code
            res.add_headers(e.headers)
            res.body = e

        elif isinstance(e, NotImplementedError):
            logger.warning(str(e), exc_info=True)
            res.code = 501
            res.body = e

        elif isinstance(e, TypeError):
            e_msg = unicode(e)
            if e_msg.startswith(req.method) and 'argument' in e_msg:
                logger.debug(e_msg, exc_info=True)
                logger.warning(
                    " ".join([
                        "Either the path arguments ({} args) or the keyword arguments",
                        "({} args) for {}.{} do not match the {} handling method's",
                        "definition"
                    ]).format(
                        len(req.controller_info["method_args"]),
                        len(req.controller_info["method_kwargs"]),
                        req.controller_info['module_name'],
                        req.controller_info['class_name'],
                        req.method
                    )
                )
                res.code = 405

            else:
                logger.exception(e)
                res.code = 500

            res.body = e

        else:
            logger.exception(e)
            res.code = 500
            res.body = e

        if con:
            error_method = getattr(con, "handle_{}_error".format(req.method), None)
            if not error_method:
                error_method = getattr(con, "handle_error")

            logger.debug("Using error method: {}.{}".format(
                con.__class__.__name__,
                error_method.__name__
            ))
            error_method(e, **kwargs)

Response: This method will set the response body and code to the error code of the current resource and set the response body to the error code of the current resource.


Instruction:
Make a summary of the following Python 3 code
def module_names(self):
        controller_prefix = self.controller_prefix
        _module_name_cache = self._module_name_cache
        if controller_prefix in _module_name_cache:
            return _module_name_cache[controller_prefix]

        module = self.get_module(controller_prefix)

        if hasattr(module, "__path__"):
            # path attr exists so this is a package
            modules = self.find_modules(module.__path__[0], controller_prefix)

        else:
            # we have a lonely .py file
            modules = set([controller_prefix])

        _module_name_cache.setdefault(controller_prefix, {})
        _module_name_cache[controller_prefix] = modules

        return modules

Response: get all the modules in the controller_prefix


Instruction:
Can you generate a brief explanation for the following Python 3 code
def modules(self):
        for modname in self.module_names:
            module = importlib.import_module(modname)
            yield module

Response: Returns an iterator of the actual modules not just their names


Instruction:
Can you generate the documentation for the following Python 3 function
def find_modules(self, path, prefix):

        modules = set([prefix])

        # https://docs.python.org/2/library/pkgutil.html#pkgutil.iter_modules
        for module_info in pkgutil.iter_modules([path]):
            # we want to ignore any "private" modules
            if module_info[1].startswith('_'): continue

            module_prefix = ".".join([prefix, module_info[1]])
            if module_info[2]:
                # module is a package
                submodules = self.find_modules(os.path.join(path, module_info[1]), module_prefix)
                modules.update(submodules)
            else:
                modules.add(module_prefix)

        return modules

Response:
recursive method that will find all the submodules of the given module
            at the given path with prefix with path


Instruction:
Given the following Python 3 function, write the documentation
def get_module_name(self, path_args):
        controller_prefix = self.controller_prefix
        cset = self.module_names
        module_name = controller_prefix
        mod_name = module_name
        while path_args:
            mod_name += "." + path_args[0]
            if mod_name in cset:
                module_name = mod_name
                path_args.pop(0)
            else:
                break

        return module_name, path_args

Response: returns the module_name and remaining path args.


Instruction:
How would you explain what the following Python 3 function does
def get_class(self, module, class_name):
        # let's get the class
        class_object = getattr(module, class_name, None)
        if not class_object or not issubclass(class_object, Controller):
            class_object = None

        return class_object

Response:
try and get the class_name from the module and make sure it is a valid
        controller


Instruction:
Can you tell what is the following Python 3 function doing
def OPTIONS(self, *args, **kwargs):
        if not self.cors:
            raise CallError(405)

        req = self.request

        origin = req.get_header('origin')
        if not origin:
            raise CallError(400, 'Need Origin header')
        call_headers = [
            ('Access-Control-Request-Headers', 'Access-Control-Allow-Headers'),
            ('Access-Control-Request-Method', 'Access-Control-Allow-Methods')
        ]
        for req_header, res_header in call_headers:
            v = req.get_header(req_header)
            if v:
                self.response.set_header(res_header, v)
            else:
                raise CallError(400, 'Need {} header'.format(req_header))

        other_headers = {
            'Access-Control-Allow-Credentials': 'true',
            'Access-Control-Max-Age': 3600
        }
        self.response.add_headers(other_headers)

Response: Handles CORS requests for this controller.


Instruction:
Implement a Python 3 function for
handling the request and returns the response

Response:
def handle(self, *controller_args, **controller_kwargs):
        """handles the request and returns the response

        This should set any response information directly onto self.response

        this method has the same signature as the request handling methods
        (eg, GET, POST) so subclasses can override this method and add decorators

        :param *controller_args: tuple, the path arguments that will be passed to
            the request handling method (eg, GET, POST)
        :param **controller_kwargs: dict, the query and body params merged together
        """
        req = self.request
        res = self.response
        res.set_header('Content-Type', "{};charset={}".format(
            self.content_type,
            self.encoding
        ))

        encoding = req.accept_encoding
        res.encoding = encoding if encoding else self.encoding

        res_method_name = ""
        controller_methods = self.find_methods()
        #controller_args, controller_kwargs = self.find_method_params()
        for controller_method_name, controller_method in controller_methods:
            try:
                logger.debug("Attempting to handle request with {}.{}.{}".format(
                    req.controller_info['module_name'],
                    req.controller_info['class_name'],
                    controller_method_name
                ))
                res.body = controller_method(
                    *controller_args,
                    **controller_kwargs
                )
                res_method_name = controller_method_name
                break

            except VersionError as e:
                logger.debug("Request {}.{}.{} failed version check [{} not in {}]".format(
                    req.controller_info['module_name'],
                    req.controller_info['class_name'],
                    controller_method_name,
                    e.request_version,
                    e.versions
                ))

            except RouteError:
                logger.debug("Request {}.{}.{} failed routing check".format(
                    req.controller_info['module_name'],
                    req.controller_info['class_name'],
                    controller_method_name
                ))

        if not res_method_name:
            # https://www.w3.org/Protocols/rfc2616/rfc2616-sec5.html#sec5.1
            # An origin server SHOULD return the status code 405 (Method Not Allowed)
            # if the method is known by the origin server but not allowed for the
            # requested resource
            raise CallError(405, "Could not find a method to satisfy {}".format(
                req.path
            ))


Instruction:
Can you generate the documentation for the following Python 3 function
def find_methods(self):
        methods = []
        req = self.request
        method_name = req.method.upper()
        method_names = set()

        members = inspect.getmembers(self)
        for member_name, member in members:
            if member_name.startswith(method_name):
                if member:
                    methods.append((member_name, member))
                    method_names.add(member_name)

        if len(methods) == 0:
            # https://www.w3.org/Protocols/rfc2616/rfc2616-sec5.html#sec5.1
            # and 501 (Not Implemented) if the method is unrecognized or not
            # implemented by the origin server
            logger.warning("No methods to handle {} found".format(method_name), exc_info=True)
            raise CallError(501, "{} {} not implemented".format(req.method, req.path))

        elif len(methods) > 1 and method_name in method_names:
            raise ValueError(
                " ".join([
                    "A multi method {} request should not have any methods named {}.",
                    "Instead, all {} methods should use use an appropriate decorator",
                    "like @route or @version and have a unique name starting with {}_"
                ]).format(
                    method_name,
                    method_name,
                    method_name,
                    method_name
                )
            )

        return methods

Response: Find the methods that could satisfy this request. This will go through and find any methods that start with the request. method GET or POST. This will find any methods that start with the request. method GET or POST. This will find any methods that start with the request. method GET or POST.


Instruction:
Can you implement a function in Python 3 that
returns the method params

Response:
def find_method_params(self):
        """Return the method params

        :returns: tuple (args, kwargs) that will be passed as *args, **kwargs
        """
        req = self.request
        args = req.controller_info["method_args"]
        kwargs = req.controller_info["method_kwargs"]
        return args, kwargs


Instruction:
Can you tell what is the following Python 3 function doing
def log_start(self, start):
        if not logger.isEnabledFor(logging.INFO): return

        try:
            req = self.request

            logger.info("REQUEST {} {}?{}".format(req.method, req.path, req.query))
            logger.info(datetime.datetime.strftime(datetime.datetime.utcnow(), "DATE %Y-%m-%dT%H:%M:%S.%f"))

            ip = req.ip
            if ip:
                logger.info("\tIP ADDRESS: {}".format(ip))

            if 'authorization' in req.headers:
                logger.info('AUTH {}'.format(req.headers['authorization']))

            ignore_hs = set([
                'accept-language',
                'accept-encoding',
                'connection',
                'authorization',
                'host',
                'x-forwarded-for'
            ])
            hs = ["Request Headers..."]
            for k, v in req.headers.items():
                if k not in ignore_hs:
                    hs.append("\t{}: {}".format(k, v))

            logger.info(os.linesep.join(hs))

        except Exception as e:
            logger.warn(e, exc_info=True)

Response: log all the headers and stuff at the start of the request


Instruction:
How would you explain what the following Python 3 function does
def log_stop(self, start):
        if not logger.isEnabledFor(logging.INFO): return

        stop = time.time()
        get_elapsed = lambda start, stop, multiplier, rnd: round(abs(stop - start) * float(multiplier), rnd)
        elapsed = get_elapsed(start, stop, 1000.00, 1)
        total = "%0.1f ms" % (elapsed)
        logger.info("RESPONSE {} {} in {}".format(self.response.code, self.response.status, total))

Response: log a summary line on how the request went


Instruction:
Explain what the following Python 3 code does
def build_lane_from_yaml(path):
    # Open
    with open(path, 'rb') as yaml_definition:
        definition = yaml.load(yaml_definition)

    # Validate schema
    try:
        validate_schema(definition)
    except SchemaError as exc:
        raise LaneSchemaError(**exc.__dict__)

    def build(lb_def, branch=False):
        """Function to recursively build the `sparklanes.Lane` object from a YAML definition"""
        init_kwargs = {k: lb_def[k] for k in (a for a in ('run_parallel', 'name') if a in lb_def)}
        lane_or_branch = Lane(**init_kwargs) if not branch else Branch(**init_kwargs)

        for task in lb_def['tasks']:
            if 'branch' in task:
                branch_def = task['branch']
                lane_or_branch.add(build(branch_def, True))
            else:
                sep = task['class'].rfind('.')
                if sep == -1:
                    raise LaneImportError('Class must include its parent module')
                mdl = task['class'][:sep]
                cls_ = task['class'][sep + 1:]

                try:
                    cls = getattr(import_module(mdl), cls_)
                except ImportError:
                    raise LaneImportError('Could not find module %s' % mdl)
                except AttributeError:
                    raise LaneImportError('Could not find class %s' % cls_)

                args = task['args'] if 'args' in task else []
                args = [args] if not isinstance(args, list) else args
                kwargs = task['kwargs'] if 'kwargs' in task else {}
                lane_or_branch.add(cls, *args, **kwargs)

        return lane_or_branch

    return build(definition['lane'])

Response: Builds a Sparklanes. Lane object from a YAML file.


Instruction:
Write a Python 3 script to
check if a class is a task and if it has been decorated with Sparklanes. Task and if it has the supplied args and kwargs match the signature of the task s entry method.

Response:
def __validate_task(self, cls, entry_mtd_name, args, kwargs):
        """Checks if a class is a task, i.e. if it has been decorated with `sparklanes.Task`, and if
        the supplied args/kwargs match the signature of the task's entry method.

        Parameters
        ----------
        cls : LaneTask
        entry_mtd_name : str
            Name of the method, which is called when the task is run
        args : list
        kwargs : dict
        """
        if not isclass(cls) or not issubclass(cls, LaneTask):
            raise TypeError('Tried to add non-Task `%s` to a Lane. Are you sure the task was '
                            'decorated with `sparklanes.Task`?' % str(cls))

        validate_params(cls, entry_mtd_name, *args, **kwargs)


Instruction:
Make a summary of the following Python 3 code
def add(self, cls_or_branch, *args, **kwargs):
        if isinstance(cls_or_branch, Branch):
            self.tasks.append(cls_or_branch)  # Add branch with already validated tasks
        else:
            # Validate
            self.__validate_task(cls_or_branch, '__init__', args, kwargs)
            # Append
            self.tasks.append({'cls_or_branch': cls_or_branch, 'args': args, 'kwargs': kwargs})

        return self

Response: Adds a task or branch to the lane.


Instruction:
Implement a Python 3 function for
executing the tasks in the lane in the order in which they have been added.

Response:
def run(self):
        """Executes the tasks in the lane in the order in which they have been added, unless
        `self.run_parallel` is True, then a thread is spawned for each task and executed in
        parallel (note that task threads are still spawned in the order in which they were added).
        """
        logger = make_default_logger(INTERNAL_LOGGER_NAME)
        logger.info('\n%s\nExecuting `%s`\n%s\n', '-'*80, self.name, '-'*80)
        logger.info('\n%s', str(self))

        threads = []

        if not self.tasks:
            raise LaneExecutionError('No tasks to execute!')

        for task_def_or_branch in self.tasks:
            if isinstance(task_def_or_branch, Branch):
                task_def_or_branch.run()
            elif isinstance(task_def_or_branch['cls_or_branch'], Branch):  # Nested Branch
                task_def_or_branch['cls_or_branch'].run()
            else:
                task = task_def_or_branch['cls_or_branch'](*task_def_or_branch['args'],
                                                           **task_def_or_branch['kwargs'])
                if self.run_parallel:
                    threads.append(LaneTaskThread(task))
                else:
                    task()

        if threads:
            for thread in threads:
                thread.start()
            for thread in threads:
                thread.join()

        logger.info('\n%s\nFinished executing `%s`\n%s', '-'*80, self.name, '-'*80)

        return self


Instruction:
How would you code a function in Python 3 to
get the historical tank data.

Response:
def get_historical_data(nmr_problems):
    """Get the historical tank data.

    Args:
        nmr_problems (int): the number of problems

    Returns:
        tuple: (observations, nmr_tanks_ground_truth)
    """
    observations = np.tile(np.array([[10, 256, 202, 97]]), (nmr_problems, 1))
    nmr_tanks_ground_truth = np.ones((nmr_problems,)) * 276
    return observations, nmr_tanks_ground_truth


Instruction:
Can you generate the documentation for the following Python 3 function
def get_simulated_data(nmr_problems):
    # The number of tanks we observe per problem
    nmr_observed_tanks = 10

    # Generate some maximum number of tanks. Basically the ground truth of the estimation problem.
    nmr_tanks_ground_truth = normal(nmr_problems, 1, mean=250, std=30, ctype='uint')

    # Generate some random tank observations
    observations = uniform(nmr_problems, nmr_observed_tanks, low=0, high=nmr_tanks_ground_truth, ctype='uint')

    return observations, nmr_tanks_ground_truth

Response: Simulate some data. This returns the simulated tank observations and the corresponding ground truth maximum number of tanks.


Instruction:
How would you code a function in Python 3 to
estimate and return the upper triangular elements of the Hessian of the given function at the given parameters.

Response:
def estimate_hessian(objective_func, parameters,
                     lower_bounds=None, upper_bounds=None,
                     step_ratio=2, nmr_steps=5,
                     max_step_sizes=None,
                     data=None, cl_runtime_info=None):
    """Estimate and return the upper triangular elements of the Hessian of the given function at the given parameters.

    This calculates the Hessian using central difference (using a 2nd order Taylor expansion) with a Richardson
    extrapolation over the proposed sequence of steps. If enough steps are given, we apply a Wynn epsilon extrapolation
    on top of the Richardson extrapolated results. If more steps are left, we return the estimate with the lowest error,
    taking into account outliers using a median filter.

    The Hessian is evaluated at the steps:

    .. math::
        \quad  ((f(x + d_j e_j + d_k e_k) - f(x + d_j e_j - d_k e_k)) -
                (f(x - d_j e_j + d_k e_k) - f(x - d_j e_j - d_k e_k)) /
                (4 d_j d_k)

    where :math:`e_j` is a vector where element :math:`j` is one and the rest are zero
    and :math:`d_j` is a scalar spacing :math:`steps_j`.

    Steps are generated according to an exponentially diminishing ratio, defined as:

        steps = max_step * step_ratio**-i, i = 0,1,..,nmr_steps-1.

    Where the maximum step can be provided. For example, a maximum step of 2 with a step ratio of 2, computed for
    4 steps gives: [2.0, 1.0, 0.5, 0.25]. If lower and upper bounds are given, we use as maximum step size the largest
    step size that fits between the Hessian point and the boundaries.

    The steps define the order of the estimation, with 2 steps resulting in a O(h^2) estimate, 3 steps resulting in a
    O(h^4) estimate and 4 or more steps resulting in a O(h^6) derivative estimate.

    Args:
        objective_func (mot.lib.cl_function.CLFunction): The function we want to differentiate.
            A CL function with the signature:

            .. code-block:: c

                double <func_name>(local const mot_float_type* const x, void* data);

            The objective function has the same signature as the minimization function in MOT. For the numerical
            hessian, the ``objective_list`` parameter is ignored.

        parameters (ndarray): The parameters at which to evaluate the gradient. A (d, p) matrix with d problems,
            and p parameters
        lower_bounds (list or None): a list of length (p,) for p parameters with the lower bounds.
            Each element of the list can be a scalar or a vector (of the same length as the number
            of problem instances). To disable bounds for this parameter use -np.inf.
        upper_bounds (list or None): a list of length (p,) for p parameters with the upper bounds.
            Each element of the list can be a scalar or a vector (of the same length as the number
            of problem instances). To disable bounds for this parameter use np.inf.
        step_ratio (float): the ratio at which the steps diminish.
        nmr_steps (int): the number of steps we will generate. We will calculate the derivative for each of these
            step sizes and extrapolate the best step size from among them. The minimum number of steps is 1.
        max_step_sizes (float or ndarray or None): the maximum step size, or the maximum step size per parameter.
            If None is given, we use 0.1 for all parameters. If a float is given, we use that for all parameters.
            If a list is given, it should be of the same length as the number of parameters.
        data (mot.lib.kernel_data.KernelData): the user provided data for the ``void* data`` pointer.
        cl_runtime_info (mot.configuration.CLRuntimeInfo): the runtime information

    Returns:
        ndarray: per problem instance a vector with the upper triangular elements of the Hessian matrix.
            This array can hold NaN's, for elements where the Hessian failed to approximate.
    """
    if len(parameters.shape) == 1:
        parameters = parameters[None, :]

    nmr_voxels = parameters.shape[0]
    nmr_params = parameters.shape[1]
    nmr_derivatives = nmr_params * (nmr_params + 1) // 2

    initial_step = _get_initial_step(parameters, lower_bounds, upper_bounds, max_step_sizes)

    kernel_data = {
        'parameters': Array(parameters, ctype='mot_float_type'),
        'initial_step': Array(initial_step, ctype='float'),
        'derivatives': Zeros((nmr_voxels, nmr_derivatives), 'double'),
        'errors': Zeros((nmr_voxels, nmr_derivatives), 'double'),
        'x_tmp': LocalMemory('mot_float_type', nmr_params),
        'data': data,
        'scratch': LocalMemory('double', nmr_steps + (nmr_steps - 1) + nmr_steps)
    }

    hessian_kernel = SimpleCLFunction.from_string('''
        void _numdiff_hessian(
                global mot_float_type* parameters,
                global float* initial_step,
                global double* derivatives,
                global double* errors,
                local mot_float_type* x_tmp,
                void* data,
                local double* scratch){

            if(get_local_id(0) == 0){
                for(uint i = 0; i < ''' + str(nmr_params) + '''; i++){
                    x_tmp[i] = parameters[i];
                }
            }
            barrier(CLK_LOCAL_MEM_FENCE);

            double f_x_input = ''' + objective_func.get_cl_function_name() + '''(x_tmp, data);

            // upper triangle loop
            uint coord_ind = 0;
            for(int i = 0; i < ''' + str(nmr_params) + '''; i++){
                for(int j = i; j < ''' + str(nmr_params) + '''; j++){
                    _numdiff_hessian_element(
                        data, x_tmp, f_x_input, i, j, initial_step,
                        derivatives + coord_ind, errors + coord_ind, scratch);

                    coord_ind++;
                }
            }
        }
    ''', dependencies=[objective_func,
                       _get_numdiff_hessian_element_func(objective_func, nmr_steps, step_ratio)])

    hessian_kernel.evaluate(kernel_data, nmr_voxels, use_local_reduction=True, cl_runtime_info=cl_runtime_info)

    return kernel_data['derivatives'].get_data()


Instruction:
Write a Python 3 function that can
return a function to compute one element of the Hessian matrix.

Response:
def _get_numdiff_hessian_element_func(objective_func, nmr_steps, step_ratio):
    """Return a function to compute one element of the Hessian matrix."""
    return SimpleCLFunction.from_string('''
        /**
         * Compute the Hessian using (possibly) multiple steps with various interpolations.
         */
        void _numdiff_hessian_element(
                void* data, local mot_float_type* x_tmp, mot_float_type f_x_input,
                uint px, uint py, global float* initial_step, global double* derivative,
                global double* error, local double* scratch){

            const uint nmr_steps = ''' + str(nmr_steps) + ''';
            uint nmr_steps_remaining = nmr_steps;

            local double* scratch_ind = scratch;
            local double* steps = scratch_ind;      scratch_ind += nmr_steps;
            local double* errors = scratch_ind;     scratch_ind += nmr_steps - 1;
            local double* steps_tmp = scratch_ind;  scratch_ind += nmr_steps;

            if(get_local_id(0) == 0){
                for(int i = 0; i < nmr_steps - 1; i++){
                    errors[i] = 0;
                }
            }
            barrier(CLK_LOCAL_MEM_FENCE);

            _numdiff_hessian_steps(data, x_tmp, f_x_input, px, py, steps, initial_step);

            if(nmr_steps_remaining > 1){
                nmr_steps_remaining = _numdiff_hessian_richardson_extrapolation(steps);
                barrier(CLK_LOCAL_MEM_FENCE);
            }

            if(nmr_steps_remaining >= 3){
                nmr_steps_remaining = _numdiff_wynn_extrapolation(steps, errors, nmr_steps_remaining);
                barrier(CLK_LOCAL_MEM_FENCE);
            }

            if(nmr_steps_remaining > 1){
                _numdiff_find_best_step(steps, errors, steps_tmp, nmr_steps_remaining);
                barrier(CLK_LOCAL_MEM_FENCE);
            }

            if(get_local_id(0) == 0){
                *derivative = steps[0];
                *error = errors[0];
            }
        }
    ''', dependencies=[
        _get_numdiff_hessian_steps_func(objective_func, nmr_steps, step_ratio),
        _get_numdiff_hessian_richardson_extrapolation_func(nmr_steps, step_ratio),
        _get_numdiff_wynn_extrapolation_func(),
        _get_numdiff_find_best_step_func()
    ])


Instruction:
Make a summary of the following Python 3 code
def _get_numdiff_hessian_steps_func(objective_func, nmr_steps, step_ratio):
    return SimpleCLFunction.from_string('''
        /**
         * Compute one element of the Hessian for a number of steps.
         *
         * This uses the initial steps in the data structure, indexed by the parameters to change (px, py).
         *
         * Args:
         *  data: the data container
         *  x_tmp: the array with the input parameters, needs to be writable, although it will return
         *         the same values.
         *  f_x_input: the objective function value at the original set of parameters
         *  px: the index of the first parameter to perturbate
         *  py: the index of the second parameter to perturbate
         *  steps: storage location for the output steps
         *  initial_step: the initial steps, array of same length as x_temp
         */
        void _numdiff_hessian_steps(void* data, local mot_float_type* x_tmp,
                                    mot_float_type f_x_input,
                                    uint px, uint py,
                                    local double* steps,
                                    global float* initial_step){

            double step_x;
            double step_y;
            double tmp;
            bool is_first_workitem = get_local_id(0) == 0;

            if(px == py){
                for(uint step_ind = 0; step_ind < ''' + str(nmr_steps) + '''; step_ind++){
                    step_x = initial_step[px] / pown(''' + str(float(step_ratio)) + ''', step_ind);

                    tmp = (
                          _numdiff_hessian_eval_step_mono(data, x_tmp, px, 2 * step_x)
                        + _numdiff_hessian_eval_step_mono(data, x_tmp, px, -2 * step_x)
                        - 2 * f_x_input
                    ) / (4 * step_x * step_x);

                    if(is_first_workitem){
                        steps[step_ind] = tmp;
                    }
                }
            }
            else{
                for(uint step_ind = 0; step_ind < ''' + str(nmr_steps) + '''; step_ind++){
                    step_x = initial_step[px] / pown(''' + str(float(step_ratio)) + ''', step_ind);
                    step_y = initial_step[py] / pown(''' + str(float(step_ratio)) + ''', step_ind);

                    tmp = (
                          _numdiff_hessian_eval_step_bi(data, x_tmp, px, step_x, py, step_y)
                        - _numdiff_hessian_eval_step_bi(data, x_tmp, px, step_x, py, -step_y)
                        - _numdiff_hessian_eval_step_bi(data, x_tmp, px, -step_x, py, step_y)
                        + _numdiff_hessian_eval_step_bi(data, x_tmp, px, -step_x, py, -step_y)
                    ) / (4 * step_x * step_y);

                    if(is_first_workitem){
                        steps[step_ind] = tmp;
                    }
                }
            }
        }
    ''', dependencies=[SimpleCLFunction.from_string('''
        /**
         * Evaluate the model with a perturbation in one dimensions.
         *
         * Args:
         *  data: the data container
         *  x_tmp: the array with the input parameters, needs to be writable, although it will return
         *         the same values.
         *  perturb_dim0: the index (into the x_tmp parameters) of the parameter to perturbate
         *  perturb_0: the added perturbation of the index corresponding to ``perturb_dim_0``
         *
         * Returns:
         *  the function evaluated at the parameters plus their perturbation.
         */
        double _numdiff_hessian_eval_step_mono(
                void* data, local mot_float_type* x_tmp,
                uint perturb_dim_0, mot_float_type perturb_0){

            mot_float_type old_0;
            double return_val;

            if(get_local_id(0) == 0){
                old_0 = x_tmp[perturb_dim_0];
                x_tmp[perturb_dim_0] += perturb_0;
            }
            barrier(CLK_LOCAL_MEM_FENCE);

            return_val = ''' + objective_func.get_cl_function_name() + '''(x_tmp, data);
            barrier(CLK_LOCAL_MEM_FENCE);

            if(get_local_id(0) == 0){
                x_tmp[perturb_dim_0] = old_0;
            }
            barrier(CLK_LOCAL_MEM_FENCE);

            return return_val;
        }
    '''), SimpleCLFunction.from_string('''
        /**
         * Evaluate the model with a perturbation in two dimensions.
         *
         * Args:
         *  data: the data container
         *  x_tmp: the array with the input parameters, needs to be writable, although it will return
         *         the same values.
         *  perturb_dim_0: the index (into the x_tmp parameters) of the first parameter to perturbate
         *  perturb_0: the added perturbation of the index corresponding to ``perturb_dim_0``
         *  perturb_dim_1: the index (into the x_tmp parameters) of the second parameter to perturbate
         *  perturb_1: the added perturbation of the index corresponding to ``perturb_dim_1``
         *
         * Returns:
         *  the function evaluated at the parameters plus their perturbation.
         */
        double _numdiff_hessian_eval_step_bi(
                void* data, local mot_float_type* x_tmp,
                uint perturb_dim_0, mot_float_type perturb_0,
                uint perturb_dim_1, mot_float_type perturb_1){

            mot_float_type old_0;
            mot_float_type old_1;
            double return_val;

            if(get_local_id(0) == 0){
                old_0 = x_tmp[perturb_dim_0];
                old_1 = x_tmp[perturb_dim_1];

                x_tmp[perturb_dim_0] += perturb_0;
                x_tmp[perturb_dim_1] += perturb_1;
            }
            barrier(CLK_LOCAL_MEM_FENCE);

            return_val = ''' + objective_func.get_cl_function_name() + '''(x_tmp, data);
            barrier(CLK_LOCAL_MEM_FENCE);

            if(get_local_id(0) == 0){
                x_tmp[perturb_dim_0] = old_0;
                x_tmp[perturb_dim_1] = old_1;
            }
            barrier(CLK_LOCAL_MEM_FENCE);

            return return_val;
        }
    ''')])

Response: Get a function to compute the multiple step sizes for a single element of the Hessian.


Instruction:
Write a Python 3 script to
get an initial step size to use for every parameter. This chooses the step sizes based on the maximum step size and the lower and upper bounds. Args: parameters (ndarray): The parameters at which to evaluate the gradient. A (d, p) matrix with d problems, p parameters and n samples. lower_bounds (list): lower bounds upper_bounds (list): upper bounds max_step_sizes (list or None): the maximum step size, or the maximum step size per parameter. Defaults to 0.1 Returns: ndarray: for every problem instance the vector with the initial step size for each parameter.

Response:
def _get_initial_step(parameters, lower_bounds, upper_bounds, max_step_sizes):
    """Get an initial step size to use for every parameter.

    This chooses the step sizes based on the maximum step size and the lower and upper bounds.

    Args:
        parameters (ndarray): The parameters at which to evaluate the gradient. A (d, p) matrix with d problems,
            p parameters and n samples.
        lower_bounds (list): lower bounds
        upper_bounds (list): upper bounds
        max_step_sizes (list or None): the maximum step size, or the maximum step size per parameter. Defaults to 0.1

    Returns:
        ndarray: for every problem instance the vector with the initial step size for each parameter.
    """
    nmr_params = parameters.shape[1]

    initial_step = np.zeros_like(parameters)

    if max_step_sizes is None:
        max_step_sizes = 0.1
    if isinstance(max_step_sizes, Number):
        max_step_sizes = [max_step_sizes] * nmr_params
    max_step_sizes = np.array(max_step_sizes)

    for ind in range(parameters.shape[1]):
        minimum_allowed_step = np.minimum(np.abs(parameters[:, ind] - lower_bounds[ind]),
                                          np.abs(upper_bounds[ind] - parameters[:, ind]))
        initial_step[:, ind] = np.minimum(minimum_allowed_step, max_step_sizes[ind])

    return initial_step / 2.


Instruction:
Explain what the following Python 3 code does
def apply(self):
        self._old_config = {k: v for k, v in _config.items()}
        self._apply()

Response: Apply the current action to the current runtime configuration.


Instruction:
Implement a Python 3 function for
resetting the current configuration to the previous state.

Response:
def unapply(self):
        """Reset the current configuration to the previous state."""
        for key, value in self._old_config.items():
            _config[key] = value


Instruction:
Can you write a function in Python 3 where it
assigns an attribute reference to all subsequent tasks.

Response:
def cache(self, name, val, overwrite=True):
        """Assigns an attribute reference to all subsequent tasks. For example, if a task caches a
        DataFrame `df` using `self.cache('some_df', df)`, all tasks that follow can access the
        DataFrame using `self.some_df`. Note that manually assigned attributes that share the same
        name have precedence over cached attributes.

        Parameters
        ----------
        name : str
            Name of the attribute
        val
            Attribute value
        overwrite : bool
            Indicates if the attribute shall be overwritten, or not (if `False`, and
            a cached attribute with the given name already exists, `sparklanes.errors.CacheError`
            will be thrown).
        """
        if name in TaskCache.cached and not overwrite:
            raise CacheError('Object with name `%s` already in cache.' % name)
        TaskCache.cached[name] = val


Instruction:
Given the following Python 3 function, write the documentation
def run(self):
        self.exc = None
        try:
            self.task()
        except BaseException:
            self.exc = sys.exc_info()

Response:
Overwrites threading. Thread. run to allow handling of exceptions thrown by threads
            from within the main app.


Instruction:
Here you have a function in Python 3, explain what it does
def join(self, timeout=None):
        Thread.join(self, timeout=timeout)
        if self.exc:
            msg = "Thread '%s' threw an exception `%s`: %s" \
                  % (self.getName(), self.exc[0].__name__, self.exc[1])
            new_exc = LaneExecutionError(msg)

            if PY3:
                raise new_exc.with_traceback(self.exc[2])  # pylint: disable=no-member
            else:
                raise (new_exc.__class__, new_exc, self.exc[2])

Response:
Overwrites threading. Thread. join to allow handling of exceptions thrown by threads
            from within the main app.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def mock_decorator(*args, **kwargs):
    def _called_decorator(dec_func):
        @wraps(dec_func)
        def _decorator(*args, **kwargs):
            return dec_func()
        return _decorator
    return _called_decorator

Response: Mocks a function to be called by the user.


Instruction:
Write a Python 3 script to
mock all modules starting with one of the mock_modules names.

Response:
def import_mock(name, *args, **kwargs):
    """Mock all modules starting with one of the mock_modules names."""
    if any(name.startswith(s) for s in mock_modules):
        return MockModule()
    return orig_import(name, *args, **kwargs)


Instruction:
Given the following Python 3 function, write the documentation
def apply_cl_function(cl_function, kernel_data, nmr_instances, use_local_reduction=False, cl_runtime_info=None):
    cl_runtime_info = cl_runtime_info or CLRuntimeInfo()
    cl_environments = cl_runtime_info.cl_environments

    for param in cl_function.get_parameters():
        if param.name not in kernel_data:
            names = [param.name for param in cl_function.get_parameters()]
            missing_names = [name for name in names if name not in kernel_data]
            raise ValueError('Some parameters are missing an input value, '
                             'required parameters are: {}, missing inputs are: {}'.format(names, missing_names))

    if cl_function.get_return_type() != 'void':
        kernel_data['_results'] = Zeros((nmr_instances,), cl_function.get_return_type())

    workers = []
    for ind, cl_environment in enumerate(cl_environments):
        worker = _ProcedureWorker(cl_environment, cl_runtime_info.compile_flags,
                                  cl_function, kernel_data, cl_runtime_info.double_precision, use_local_reduction)
        workers.append(worker)

    def enqueue_batch(batch_size, offset):
        items_per_worker = [batch_size // len(cl_environments) for _ in range(len(cl_environments) - 1)]
        items_per_worker.append(batch_size - sum(items_per_worker))

        for ind, worker in enumerate(workers):
            worker.calculate(offset, offset + items_per_worker[ind])
            offset += items_per_worker[ind]
            worker.cl_queue.flush()

        for worker in workers:
            worker.cl_queue.finish()

        return offset

    total_offset = 0
    for batch_start, batch_end in split_in_batches(nmr_instances, 1e4 * len(workers)):
        total_offset = enqueue_batch(batch_end - batch_start, total_offset)

    if cl_function.get_return_type() != 'void':
        return kernel_data['_results'].get_data()

Response: This function will run the given function on the given set of data instances and then execute the given function on the given set of data instances.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def from_string(cls, cl_function, dependencies=()):
        return_type, function_name, parameter_list, body = split_cl_function(cl_function)
        return SimpleCLFunction(return_type, function_name, parameter_list, body, dependencies=dependencies)

Response: Parse the given string into a SimpleCLFunction object.


Instruction:
Make a summary of the following Python 3 code
def _get_parameter_signatures(self):
        declarations = []
        for p in self.get_parameters():
            new_p = p.get_renamed(p.name.replace('.', '_'))
            declarations.append(new_p.get_declaration())
        return declarations

Response: Get the list of signatures of the parameters for the CL function declaration.


Instruction:
Can you create a Python 3 function that
gets the CL code for all the dependencies.

Response:
def _get_cl_dependency_code(self):
        """Get the CL code for all the CL code for all the dependencies.

        Returns:
            str: The CL code with the actual code.
        """
        code = ''
        for d in self._dependencies:
            code += d.get_cl_code() + "\n"
        return code


Instruction:
Create a Python 3 function for
getting the list of kernel arguments for loading the kernel data elements into the kernel.

Response:
def _get_kernel_arguments(self):
        """Get the list of kernel arguments for loading the kernel data elements into the kernel.

        This will use the sorted keys for looping through the kernel input items.

        Returns:
            list of str: the list of parameter definitions
        """
        declarations = []
        for name, data in self._kernel_data.items():
            declarations.extend(data.get_kernel_parameters('_' + name))
        return declarations


Instruction:
Here you have a function in Python 3, explain what it does
def get_scalar_arg_dtypes(self):
        dtypes = []
        for name, data in self._kernel_data.items():
            dtypes.extend(data.get_scalar_arg_dtypes())
        return dtypes

Response: Get the location and types of the input scalars.


Instruction:
Write a Python 3 script to
create and initializes a new `SparkContext` (the old one will be stopped). Argument signature is copied from `pyspark.SparkContext <https://spark.apache.org/docs/latest/api/python/pyspark.html#pyspark.SparkContext>`_.

Response:
def set_sc(cls, master=None, appName=None, sparkHome=None, pyFiles=None, environment=None,
               batchSize=0, serializer=PickleSerializer(), conf=None, gateway=None, jsc=None,
               profiler_cls=BasicProfiler):
        """Creates and initializes a new `SparkContext` (the old one will be stopped).
        Argument signature is copied from `pyspark.SparkContext
        <https://spark.apache.org/docs/latest/api/python/pyspark.html#pyspark.SparkContext>`_.
        """
        if cls.sc is not None:
            cls.sc.stop()
        cls.sc = SparkContext(master, appName, sparkHome, pyFiles, environment, batchSize,
                              serializer,
                              conf, gateway, jsc, profiler_cls)
        cls.__init_spark()


Instruction:
Can you generate a brief explanation for the following Python 3 code
def set_spark(cls, master=None, appName=None, conf=None, hive_support=False):
        sess = SparkSession.builder
        if master:
            sess.master(master)
        if appName:
            sess.appName(appName)
        if conf:
            sess.config(conf=conf)
        if hive_support:
            sess.enableHiveSupport()

        cls.spark = sess.getOrCreate()

Response: Creates and initializes a new SparkSession.


Instruction:
Here you have a function in Python 3, explain what it does
def _package_and_submit(args):
    args = _parse_and_validate_args(args)

    logging.debug(args)
    dist = __make_tmp_dir()
    try:
        __package_dependencies(dist_dir=dist, additional_reqs=args['requirements'],
                               silent=args['silent'])
        __package_app(tasks_pkg=args['package'],
                      dist_dir=dist,
                      custom_main=args['main'],
                      extra_data=args['extra_data'])
        __run_spark_submit(lane_yaml=args['yaml'],
                           dist_dir=dist,
                           spark_home=args['spark_home'],
                           spark_args=args['spark_args'],
                           silent=args['silent'])

    except Exception as exc:
        __clean_up(dist)
        raise exc
    __clean_up(dist)

Response: Packages and submits a job to Spark.


Instruction:
Create a Python 3 function for
parsing and validate the given arguments.

Response:
def _parse_and_validate_args(args):
    """
    Parse and validate arguments. During validation, it is checked whether the given
    files/directories exist, while also converting relative paths to absolute ones.

    Parameters
    ----------
    args (List): Command-line arguments
    """
    class ExtendAction(argparse.Action):
        def __call__(self, parser, namespace, values, option_string=None):
            if getattr(namespace, self.dest, None) is None:
                setattr(namespace, self.dest, [])
            getattr(namespace, self.dest).extend(values)

    parser = argparse.ArgumentParser(description='Submitting a lane to spark.')
    parser.add_argument('-y', '--yaml', type=str, required=True,
                        help='Path to the yaml definition file.')
    parser.add_argument('-p', '--package', type=str, required=True,
                        help='Path to the python package containing your tasks.')
    parser.add_argument('-r', '--requirements', type=str, required=False,
                        help='Path to a `requirements.txt` specifying any additional dependencies '
                             'of your tasks.')
    parser.add_argument('-e', '--extra-data', nargs='*', required=False, action=ExtendAction,
                        help='Path to any additional files or directories that should be packaged '
                             'and sent to Spark.')
    parser.add_argument('-m', '--main', type=str, required=False,
                        help='Path to a custom main python file')
    parser.add_argument('-d', '--spark-home', type=str, required=False,
                        help='Custom path to the directory containing your Spark installation. If '
                             'none is given, sparklanes will try to use the `spark-submit` command '
                             'from your PATH')
    parser.add_argument('-s', '--spark-args', nargs='*', required=False,
                        help='Any additional arguments that should be sent to Spark via '
                             'spark-submit. '
                             '(e.g. `--spark-args executor-memory=20G total-executor-cores=100`)')
    parser.add_argument('--silent', help='If set, no output will be sent to console',
                        action='store_true')
    args = parser.parse_args(args).__dict__

    # Check/fix files/dirs
    for param in ('package', 'spark_home'):
        args[param] = __validate_and_fix_path(args[param], check_dir=True)
    for param in ('yaml', 'requirements', 'main'):
        args[param] = __validate_and_fix_path(args[param], check_file=True)
    if args['extra_data']:
        for i in range(len(args['extra_data'])):
            args['extra_data'][i] = __validate_and_fix_path(args['extra_data'][i],
                                                            check_file=True, check_dir=True)

    # Check if python package
    if not os.path.isfile(os.path.join(args['package'], '__init__.py')):
        raise SystemExit('Could not confirm `%s` is a python package. Make sure it contains an '
                         '`__init__.py`.')

    # Check/fix spark args
    if args['spark_args']:
        args['spark_args'] = __validate_and_fix_spark_args(args['spark_args'])

    return args


Instruction:
How would you implement a function in Python 3 that
checks if a file or directory exists and converts relative paths to absolute ones

Response:
def __validate_and_fix_path(path, check_file=False, check_dir=False):
    """Check if a file/directory exists and converts relative paths to absolute ones"""
    # pylint: disable=superfluous-parens
    if path is None:
        return path
    else:
        if not (os.path.isfile(path) if check_file else False) \
                and not (os.path.isdir(path) if check_dir else False):
            raise SystemExit('Path `%s` does not exist' % path)
        if not os.path.isabs(path):
            path = os.path.abspath(os.path.join(os.path.abspath(os.curdir), path))

    return path


Instruction:
Can you generate a brief explanation for the following Python 3 code
def __validate_and_fix_spark_args(spark_args):
    pattern = re.compile(r'[\w\-_]+=.+')
    fixed_args = []
    for arg in spark_args:
        if arg not in SPARK_SUBMIT_FLAGS:
            if not pattern.match(arg):
                raise SystemExit('Spark argument `%s` does not seem to be in the correct format '
                                 '`ARG_NAME=ARG_VAL`, and is also not recognized to be one of the'
                                 'valid spark-submit flags (%s).' % (arg, str(SPARK_SUBMIT_FLAGS)))
            eq_pos = arg.find('=')
            fixed_args.append('--' + arg[:eq_pos])
            fixed_args.append(arg[eq_pos + 1:])
        else:
            fixed_args.append('--' + arg)

    return fixed_args

Response: Validate and fix spark arguments.


Instruction:
Implement a Python 3 function for
packaging the app s dependencies from pip and packages them to spark

Response:
def __package_dependencies(dist_dir, additional_reqs, silent):
    """
    Installs the app's dependencies from pip and packages them (as zip), to be submitted to spark.

    Parameters
    ----------
    dist_dir (str): Path to directory where the packaged libs shall be located
    additional_reqs (str): Path to a requirements.txt, containing any of the app's additional
        requirements
    silent (bool): Flag indicating whether pip output should be printed to console
    """
    logging.info('Packaging dependencies')
    libs_dir = os.path.join(dist_dir, 'libs')
    if not os.path.isdir(libs_dir):
        os.mkdir(libs_dir)

    # Get requirements
    req_txt = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'requirements-submit.txt')
    with open(req_txt, 'r') as req:
        requirements = req.read().splitlines()
    if additional_reqs:
        with open(additional_reqs, 'r') as req:
            for row in req:
                requirements.append(row)

    # Remove duplicates
    requirements = list(set(requirements))

    # Install
    devnull = open(os.devnull, 'w')
    outp = {'stderr': STDOUT, 'stdout': devnull} if silent else {}
    for pkg in requirements:
        cmd = ['pip', 'install', pkg, '-t', libs_dir]
        logging.debug('Calling `%s`', str(cmd))
        call(cmd, **outp)
    devnull.close()

    # Package
    shutil.make_archive(libs_dir, 'zip', libs_dir, './')


Instruction:
Can you generate a brief explanation for the following Python 3 code
def __package_app(tasks_pkg, dist_dir, custom_main=None, extra_data=None):
    logging.info('Packaging application')

    # Package tasks
    tasks_dir_splits = os.path.split(os.path.realpath(tasks_pkg))
    shutil.make_archive(os.path.join(dist_dir, 'tasks'),
                        'zip',
                        tasks_dir_splits[0],
                        tasks_dir_splits[1])

    # Package main.py
    if custom_main is None:
        from . import _main
        main_path = _main.__file__
        if main_path[-3:] == 'pyc':
            main_path = main_path[:-1]
        shutil.copy(os.path.realpath(main_path),
                    os.path.join(dist_dir, 'main.py'))
    else:
        shutil.copy(os.path.realpath(custom_main),
                    os.path.join(dist_dir, 'main.py'))

    # Package _framework
    shutil.make_archive(os.path.join(dist_dir, '_framework'),
                        'zip',
                        os.path.join(os.path.dirname(os.path.realpath(__file__)), '..', '..'),
                        './sparklanes/')

    # Package extra data
    if extra_data:
        for dat in extra_data:
            real_path = os.path.realpath(dat)
            target = os.path.join(dist_dir, os.path.split(real_path)[1])
            if os.path.isfile(real_path):
                shutil.copy(real_path, target)
            elif os.path.isdir(real_path):
                shutil.copytree(real_path, target)
            else:
                raise IOError('File `%s` not found at `%s`.' % (dat, real_path))

Response: Packages the tasks_pkg to dist_dir. Also copies the main. py file to spark.


Instruction:
Given the following Python 3 function, write the documentation
def __run_spark_submit(lane_yaml, dist_dir, spark_home, spark_args, silent):
    # spark-submit binary
    cmd = ['spark-submit' if spark_home is None else os.path.join(spark_home, 'bin/spark-submit')]

    # Supplied spark arguments
    if spark_args:
        cmd += spark_args

    # Packaged App & lane
    cmd += ['--py-files', 'libs.zip,_framework.zip,tasks.zip', 'main.py']
    cmd += ['--lane', lane_yaml]

    logging.info('Submitting to Spark')
    logging.debug(str(cmd))

    # Submit
    devnull = open(os.devnull, 'w')
    outp = {'stderr': STDOUT, 'stdout': devnull} if silent else {}
    call(cmd, cwd=dist_dir, env=MY_ENV, **outp)
    devnull.close()

Response: Submits the packaged application to Spark using a spark - submit subprocess


Instruction:
Can you implement a function in Python 3 that
adds include guards to the given string.

Response:
def add_include_guards(cl_str, guard_name=None):
    """Add include guards to the given string.

    If you are including the same body of CL code multiple times in a Kernel, it is important to add include
    guards (https://en.wikipedia.org/wiki/Include_guard) around them to prevent the kernel from registering the function
    twice.

    Args:
        cl_str (str): the piece of CL code as a string to which we add the include guards
        guard_name (str): the name of the C pre-processor guard. If not given we use the MD5 hash of the
            given cl string.

    Returns:
        str: the same string but then with include guards around them.
    """
    if not guard_name:
        guard_name = 'GUARD_' + hashlib.md5(cl_str.encode('utf-8')).hexdigest()

    return '''
        # ifndef {guard_name}
        # define {guard_name}
        {func_str}
        # endif // {guard_name}
    '''.format(func_str=cl_str, guard_name=guard_name)


Instruction:
Can you implement a function in Python 3 that
returns the numpy datatype of the given CL data type string.

Response:
def ctype_to_dtype(cl_type, mot_float_type='float'):
    """Get the numpy dtype of the given cl_type string.

    Args:
        cl_type (str): the CL data type to match, for example 'float' or 'float4'.
        mot_float_type (str): the C name of the ``mot_float_type``. The dtype will be looked up recursively.

    Returns:
        dtype: the numpy datatype
    """
    if is_vector_ctype(cl_type):
        raw_type, vector_length = split_vector_ctype(cl_type)

        if raw_type == 'mot_float_type':
            if is_vector_ctype(mot_float_type):
                raw_type, _ = split_vector_ctype(mot_float_type)
            else:
                raw_type = mot_float_type

        vector_type = raw_type + str(vector_length)
        return getattr(cl_array.vec, vector_type)
    else:
        if cl_type == 'mot_float_type':
            cl_type = mot_float_type

        data_types = [
            ('char', np.int8),
            ('uchar', np.uint8),
            ('short', np.int16),
            ('ushort', np.uint16),
            ('int', np.int32),
            ('uint', np.uint32),
            ('long', np.int64),
            ('ulong', np.uint64),
            ('float', np.float32),
            ('double', np.float64),
        ]
        for ctype, dtype in data_types:
            if ctype == cl_type:
                return dtype


Instruction:
Can you tell what is the following Python 3 function doing
def convert_data_to_dtype(data, data_type, mot_float_type='float'):
    scalar_dtype = ctype_to_dtype(data_type, mot_float_type)

    if isinstance(data, numbers.Number):
        data = scalar_dtype(data)

    if is_vector_ctype(data_type):
        shape = data.shape
        dtype = ctype_to_dtype(data_type, mot_float_type)
        ve = np.zeros(shape[:-1], dtype=dtype)

        if len(shape) == 1:
            for vector_ind in range(shape[0]):
                ve[0][vector_ind] = data[vector_ind]
        elif len(shape) == 2:
            for i in range(data.shape[0]):
                for vector_ind in range(data.shape[1]):
                    ve[i][vector_ind] = data[i, vector_ind]
        elif len(shape) == 3:
            for i in range(data.shape[0]):
                for j in range(data.shape[1]):
                    for vector_ind in range(data.shape[2]):
                        ve[i, j][vector_ind] = data[i, j, vector_ind]

        return np.require(ve, requirements=['C', 'A', 'O'])
    return np.require(data, scalar_dtype, ['C', 'A', 'O'])

Response: Convert the given input data to the correct numpy type.


Instruction:
How would you explain what the following Python 3 function does
def split_vector_ctype(ctype):
    if not is_vector_ctype(ctype):
        raise ValueError('The given ctype is not a vector type.')
    for vector_length in [2, 3, 4, 8, 16]:
        if ctype.endswith(str(vector_length)):
            vector_str_len = len(str(vector_length))
            return ctype[:-vector_str_len], int(ctype[-vector_str_len:])

Response: Split a vector ctype into a raw ctype and the vector length.


Instruction:
Make a summary of the following Python 3 code
def device_type_from_string(cl_device_type_str):
    cl_device_type_str = cl_device_type_str.upper()
    if hasattr(cl.device_type, cl_device_type_str):
        return getattr(cl.device_type, cl_device_type_str)
    return None

Response: Converts a string like gpu to a pyopencl device type string.


Instruction:
How would you code a function in Python 3 to
get the model floating point type definition.

Response:
def get_float_type_def(double_precision, include_complex=True):
    """Get the model floating point type definition.

    Args:
        double_precision (boolean): if True we will use the double type for the mot_float_type type.
            Else, we will use the single precision float type for the mot_float_type type.
        include_complex (boolean): if we include support for complex numbers

    Returns:
        str: defines the mot_float_type types, the epsilon and the MIN and MAX values.
    """
    if include_complex:
        with open(os.path.abspath(resource_filename('mot', 'data/opencl/complex.h')), 'r') as f:
            complex_number_support = f.read()
    else:
        complex_number_support = ''

    scipy_constants = '''
        #define MACHEP DBL_EPSILON
        #define MAXLOG log(DBL_MAX)
        #define LANCZOS_G 6.024680040776729583740234375 /* taken from Scipy */
        #define EULER 0.577215664901532860606512090082402431 /* Euler constant, from Scipy */
    '''

    if double_precision:
        return '''
            #if __OPENCL_VERSION__ <= CL_VERSION_1_1
                #pragma OPENCL EXTENSION cl_khr_fp64 : enable
            #endif

            #define PYOPENCL_DEFINE_CDOUBLE

            typedef double mot_float_type;
            typedef double2 mot_float_type2;
            typedef double4 mot_float_type4;
            typedef double8 mot_float_type8;
            typedef double16 mot_float_type16;

            #define MOT_EPSILON DBL_EPSILON
            #define MOT_MIN DBL_MIN
            #define MOT_MAX DBL_MAX
        ''' + scipy_constants + complex_number_support
    else:
        return '''
            #if __OPENCL_VERSION__ <= CL_VERSION_1_1
                #pragma OPENCL EXTENSION cl_khr_fp64 : enable
            #endif

            typedef float mot_float_type;
            typedef float2 mot_float_type2;
            typedef float4 mot_float_type4;
            typedef float8 mot_float_type8;
            typedef float16 mot_float_type16;

            #define MOT_EPSILON FLT_EPSILON
            #define MOT_MIN FLT_MIN
            #define MOT_MAX FLT_MAX
        ''' + scipy_constants + complex_number_support


Instruction:
Here you have a function in Python 3, explain what it does
def topological_sort(data):

    def check_self_dependencies(input_data):
        """Check if there are self dependencies within a node.

        Self dependencies are for example: ``{'a': ('a',)}``.

        Args:
            input_data (dict): the input data. Of a structure similar to {key: (list of values), ...}.

        Raises:
            ValueError: if there are indeed self dependencies
        """
        for k, v in input_data.items():
            if k in v:
                raise ValueError('Self-dependency, {} depends on itself.'.format(k))

    def prepare_input_data(input_data):
        """Prepares the input data by making sets of the dependencies. This automatically removes redundant items.

        Args:
            input_data (dict): the input data. Of a structure similar to {key: (list of values), ...}.

        Returns:
            dict: a copy of the input dict but with sets instead of lists for the dependencies.
        """
        return {k: set(v) for k, v in input_data.items()}

    def find_items_without_dependencies(input_data):
        """This searches the dependencies of all the items for items that have no dependencies.

        For example, suppose the input is: ``{'a': ('b',)}``, then ``a`` depends on ``b`` and ``b`` depends on nothing.
        This class returns ``(b,)`` in this example.

        Args:
            input_data (dict): the input data. Of a structure similar to {key: (list of values), ...}.

        Returns:
            list: the list of items without any dependency.
        """
        return list(reduce(set.union, input_data.values()) - set(input_data.keys()))

    def add_empty_dependencies(data):
        items_without_dependencies = find_items_without_dependencies(data)
        data.update({item: set() for item in items_without_dependencies})

    def get_sorted(input_data):
        data = input_data
        while True:
            ordered = set(item for item, dep in data.items() if len(dep) == 0)
            if not ordered:
                break
            yield ordered
            data = {item: (dep - ordered) for item, dep in data.items() if item not in ordered}

        if len(data) != 0:
            raise ValueError('Cyclic dependencies exist '
                             'among these items: {}'.format(', '.join(repr(x) for x in data.items())))

    check_self_dependencies(data)

    if not len(data):
        return []

    data_copy = prepare_input_data(data)
    add_empty_dependencies(data_copy)

    result = []
    for d in get_sorted(data_copy):
        try:
            d = sorted(d)
        except TypeError:
            d = list(d)

        result.extend(d)
    return result

Response: Topological sort the given dictionary structure.


Instruction:
Can you create a Python 3 function that
tests if the given value is a scalar.

Response:
def is_scalar(value):
    """Test if the given value is a scalar.

    This function also works with memory mapped array values, in contrast to the numpy is_scalar method.

    Args:
        value: the value to test for being a scalar value

    Returns:
        boolean: if the given value is a scalar or not
    """
    return np.isscalar(value) or (isinstance(value, np.ndarray) and (len(np.squeeze(value).shape) == 0))


Instruction:
Write a Python 3 script to
check if all elements in the given value are equal to each other.

Response:
def all_elements_equal(value):
    """Checks if all elements in the given value are equal to each other.

    If the input is a single value the result is trivial. If not, we compare all the values to see
    if they are exactly the same.

    Args:
        value (ndarray or number): a numpy array or a single number.

    Returns:
        bool: true if all elements are equal to each other, false otherwise
    """
    if is_scalar(value):
        return True
    return np.array(value == value.flatten()[0]).all()


Instruction:
Create a Python 3 function for
getting a single value out of the given value.

Response:
def get_single_value(value):
    """Get a single value out of the given value.

    This is meant to be used after a call to :func:`all_elements_equal` that returned True. With this
    function we return a single number from the input value.

    Args:
        value (ndarray or number): a numpy array or a single number.

    Returns:
        number: a single number from the input

    Raises:
        ValueError: if not all elements are equal
    """
    if not all_elements_equal(value):
        raise ValueError('Not all values are equal to each other.')

    if is_scalar(value):
        return value
    return value.item(0)


Instruction:
How would you implement a function in Python 3 that
disables all logging temporarily. A context manager that will prevent any logging messages triggered during the body from being processed. Args: highest_level: the maximum logging level that is being blocked

Response:
def all_logging_disabled(highest_level=logging.CRITICAL):
    """Disable all logging temporarily.

    A context manager that will prevent any logging messages triggered during the body from being processed.

    Args:
        highest_level: the maximum logging level that is being blocked
    """
    previous_level = logging.root.manager.disable
    logging.disable(highest_level)
    try:
        yield
    finally:
        logging.disable(previous_level)


Instruction:
Can you implement a function in Python 3 that
splits the total number of elements into batches of the specified maximum size.

Response:
def split_in_batches(nmr_elements, max_batch_size):
    """Split the total number of elements into batches of the specified maximum size.

    Examples::
        split_in_batches(30, 8) -> [(0, 8), (8, 15), (16, 23), (24, 29)]

        for batch_start, batch_end in split_in_batches(2000, 100):
            array[batch_start:batch_end]

    Yields:
        tuple: the start and end point of the next batch
    """
    offset = 0
    elements_left = nmr_elements
    while elements_left > 0:
        next_batch = (offset, offset + min(elements_left, max_batch_size))
        yield next_batch

        batch_size = min(elements_left, max_batch_size)
        elements_left -= batch_size
        offset += batch_size


Instruction:
Write a Python 3 function for
transforming a covariance matrix into a correlations matrix.

Response:
def covariance_to_correlations(covariance):
    """Transform a covariance matrix into a correlations matrix.

    This can be seen as dividing a covariance matrix by the outer product of the diagonal.

    As post processing we replace the infinities and the NaNs with zeros and clip the result to [-1, 1].

    Args:
        covariance (ndarray): a matrix of shape (n, p, p) with for n problems the covariance matrix of shape (p, p).

    Returns:
        ndarray: the correlations matrix
    """
    diagonal_ind = np.arange(covariance.shape[1])
    diagonal_els = covariance[:, diagonal_ind, diagonal_ind]
    result = covariance / np.sqrt(diagonal_els[:, :, None] * diagonal_els[:, None, :])
    result[np.isinf(result)] = 0
    return np.clip(np.nan_to_num(result), -1, 1)


Instruction:
Make a summary of the following Python 3 code
def multiprocess_mapping(func, iterable):
    if os.name == 'nt':  # In Windows there is no fork.
        return list(map(func, iterable))
    try:
        p = multiprocessing.Pool()
        return_data = list(p.imap(func, iterable))
        p.close()
        p.join()
        return return_data
    except OSError:
        return list(map(func, iterable))

Response: Multiprocess mapping the given function on the given iterable.


Instruction:
How would you explain what the following Python 3 function does
def parse_cl_function(cl_code, dependencies=()):
    from mot.lib.cl_function import SimpleCLFunction

    def separate_cl_functions(input_str):
        """Separate all the OpenCL functions.

        This creates a list of strings, with for each function found the OpenCL code.

        Args:
            input_str (str): the string containing one or more functions.

        Returns:
            list: a list of strings, with one string per found CL function.
        """
        class Semantics:

            def __init__(self):
                self._functions = []

            def result(self, ast):
                return self._functions

            def arglist(self, ast):
                return '({})'.format(', '.join(ast))

            def function(self, ast):
                def join(items):
                    result = ''
                    for item in items:
                        if isinstance(item, str):
                            result += item
                        else:
                            result += join(item)
                    return result

                self._functions.append(join(ast).strip())
                return ast

        return _extract_cl_functions_parser.parse(input_str, semantics=Semantics())

    functions = separate_cl_functions(cl_code)
    return SimpleCLFunction.from_string(functions[-1], dependencies=list(dependencies or []) + [
        SimpleCLFunction.from_string(s) for s in functions[:-1]])

Response: Parse the given OpenCL string to a single SimpleCLFunction.


Instruction:
Can you create a Python 3 function that
splits a CL function into a return type function name parameter list and body.

Response:
def split_cl_function(cl_str):
    """Split an CL function into a return type, function name, parameters list and the body.

    Args:
        cl_str (str): the CL code to parse and plit into components

    Returns:
        tuple: string elements for the return type, function name, parameter list and the body
    """
    class Semantics:

        def __init__(self):
            self._return_type = ''
            self._function_name = ''
            self._parameter_list = []
            self._cl_body = ''

        def result(self, ast):
            return self._return_type, self._function_name, self._parameter_list, self._cl_body

        def address_space(self, ast):
            self._return_type = ast.strip() + ' '
            return ast

        def data_type(self, ast):
            self._return_type += ''.join(ast).strip()
            return ast

        def function_name(self, ast):
            self._function_name = ast.strip()
            return ast

        def arglist(self, ast):
            if ast != '()':
                self._parameter_list = ast
            return ast

        def body(self, ast):
            def join(items):
                result = ''
                for item in items:
                    if isinstance(item, str):
                        result += item
                    else:
                        result += join(item)
                return result

            self._cl_body = join(ast).strip()[1:-1]
            return ast

    return _split_cl_function_parser.parse(cl_str, semantics=Semantics())


Instruction:
Given the following Python 3 function, write the documentation
def make_default_logger(name=INTERNAL_LOGGER_NAME, level=logging.INFO,
                        fmt='%(asctime)s - %(name)s - %(levelname)s - %(message)s'):
    """Create a logger with the default configuration"""
    logger = logging.getLogger(name)
    logger.setLevel(level)
    if not logger.handlers:
        handler = logging.StreamHandler(sys.stderr)
        handler.setLevel(level)
        formatter = logging.Formatter(fmt)
        handler.setFormatter(formatter)
        logger.addHandler(handler)

    return logger

Response: Create a logger with the default configuration


Instruction:
Write a Python 3 function for
checking if the device associated with this environment is a GPU.

Response:
def is_gpu(self):
        """Check if the device associated with this environment is a GPU.

        Returns:
            boolean: True if the device is an GPU, false otherwise.
        """
        return self._device.get_info(cl.device_info.TYPE) == cl.device_type.GPU


Instruction:
How would you explain what the following Python 3 function does
def is_cpu(self):
        return self._device.get_info(cl.device_info.TYPE) == cl.device_type.CPU

Response: Check if the device associated with this environment is a CPU.


Instruction:
Here you have a function in Python 3, explain what it does
def single_device(cl_device_type='GPU', platform=None, fallback_to_any_device_type=False):
        if isinstance(cl_device_type, str):
            cl_device_type = device_type_from_string(cl_device_type)

        device = None

        if platform is None:
            platforms = cl.get_platforms()
        else:
            platforms = [platform]

        for platform in platforms:
            devices = platform.get_devices(device_type=cl_device_type)

            for dev in devices:
                if device_supports_double(dev):
                    try:
                        env = CLEnvironment(platform, dev)
                        return [env]
                    except cl.RuntimeError:
                        pass

        if not device:
            if fallback_to_any_device_type:
                return cl.get_platforms()[0].get_devices()
            else:
                raise ValueError('No devices of the specified type ({}) found.'.format(
                    cl.device_type.to_string(cl_device_type)))

        raise ValueError('No suitable OpenCL device found.')

Response: Get a list containing a single environment for a given OpenCL device type on the given platform.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def all_devices(cl_device_type=None, platform=None):
        if isinstance(cl_device_type, str):
            cl_device_type = device_type_from_string(cl_device_type)

        runtime_list = []

        if platform is None:
            platforms = cl.get_platforms()
        else:
            platforms = [platform]

        for platform in platforms:
            if cl_device_type:
                devices = platform.get_devices(device_type=cl_device_type)
            else:
                devices = platform.get_devices()

            for device in devices:
                if device_supports_double(device):
                    env = CLEnvironment(platform, device)
                    runtime_list.append(env)

        return runtime_list

Response: Get multiple device environments optionally only of the indicated type.


Instruction:
Write a Python 3 script for
getting a list of device environments that is suitable for use in MOT. Basically this gets the total list of devices using all_devices() and applies a filter on it. This filter does the following: 1) if the 'AMD Accelerated Parallel Processing' is available remove all environments using the 'Clover' platform. More things may be implemented in the future. Args: preferred_device_type (str): the preferred device type, one of 'CPU', 'GPU' or 'APU'. If no devices of this type can be found, we will use any other device available. Returns: list of CLEnvironment: List with the CL device environments.

Response:
def smart_device_selection(preferred_device_type=None):
        """Get a list of device environments that is suitable for use in MOT.

        Basically this gets the total list of devices using all_devices() and applies a filter on it.

        This filter does the following:
            1) if the 'AMD Accelerated Parallel Processing' is available remove all environments using the 'Clover'
                platform.

        More things may be implemented in the future.

        Args:
            preferred_device_type (str): the preferred device type, one of 'CPU', 'GPU' or 'APU'.
                If no devices of this type can be found, we will use any other device available.

        Returns:
            list of CLEnvironment: List with the CL device environments.
        """
        cl_environments = CLEnvironmentFactory.all_devices(cl_device_type=preferred_device_type)
        platform_names = [env.platform.name for env in cl_environments]
        has_amd_pro_platform = any('AMD Accelerated Parallel Processing' in name for name in platform_names)

        if has_amd_pro_platform:
            return list(filter(lambda env: 'Clover' not in env.platform.name, cl_environments))

        if preferred_device_type is not None and not len(cl_environments):
            return CLEnvironmentFactory.all_devices()

        return cl_environments


Instruction:
Can you generate the documentation for the following Python 3 function
def multivariate_ess(samples, batch_size_generator=None):
    samples_generator = _get_sample_generator(samples)
    return np.array(multiprocess_mapping(_MultivariateESSMultiProcessing(batch_size_generator), samples_generator()))

Response: r Estimates the multivariate Effective Sample Size for the samples of every problem.


Instruction:
How would you explain what the following Python 3 function does
def univariate_ess(samples, method='standard_error', **kwargs):
    samples_generator = _get_sample_generator(samples)
    return np.array(multiprocess_mapping(_UnivariateESSMultiProcessing(method, **kwargs), samples_generator()))

Response: r Estimates the univariate Effective Sample Size for the samples of every problem.


Instruction:
How would you code a function in Python 3 to
get a sample generator from the given polymorphic input.

Response:
def _get_sample_generator(samples):
    """Get a sample generator from the given polymorphic input.

    Args:
        samples (ndarray, dict or generator): either an matrix of shape (d, p, n) with d problems, p parameters and
            n samples, or a dictionary with for every parameter a matrix with shape (d, n) or, finally,
            a generator function that yields sample arrays of shape (p, n).

    Returns:
        generator: a generator that yields a matrix of size (p, n) for every problem in the input.
    """
    if isinstance(samples, Mapping):
        def samples_generator():
            for ind in range(samples[list(samples.keys())[0]].shape[0]):
                yield np.array([samples[s][ind, :] for s in sorted(samples)])
    elif isinstance(samples, np.ndarray):
        def samples_generator():
            for ind in range(samples.shape[0]):
                yield samples[ind]
    else:
        samples_generator = samples
    return samples_generator


Instruction:
Given the following Python 3 function, write the documentation
def estimate_univariate_ess_standard_error(chain, batch_size_generator=None, compute_method=None):
    sigma = (monte_carlo_standard_error(chain, batch_size_generator=batch_size_generator,
                                        compute_method=compute_method) ** 2 * len(chain))
    lambda_ = np.var(chain, dtype=np.float64)
    return len(chain) * (lambda_ / sigma)

Response: r Estimates the univariate ESS using the standard error method.


Instruction:
Given the following Python 3 function, write the documentation
def minimum_multivariate_ess(nmr_params, alpha=0.05, epsilon=0.05):
    tmp = 2.0 / nmr_params
    log_min_ess = tmp * np.log(2) + np.log(np.pi) - tmp * (np.log(nmr_params) + gammaln(nmr_params / 2)) \
                  + np.log(chi2.ppf(1 - alpha, nmr_params)) - 2 * np.log(epsilon)
    return int(round(np.exp(log_min_ess)))

Response: r Calculates the minimum multivariate Effective Sample Size for a given set of free parameters.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def multivariate_ess_precision(nmr_params, multi_variate_ess, alpha=0.05):
    tmp = 2.0 / nmr_params
    log_min_ess = tmp * np.log(2) + np.log(np.pi) - tmp * (np.log(nmr_params) + gammaln(nmr_params / 2)) \
                  + np.log(chi2.ppf(1 - alpha, nmr_params)) - np.log(multi_variate_ess)
    return np.sqrt(np.exp(log_min_ess))

Response: r Calculates the precision given the multivariate Effective Sample Size and the confidence region of the MCMC sample.


Instruction:
Can you tell what is the following Python 3 function doing
def estimate_multivariate_ess_sigma(samples, batch_size):
    sample_means = np.mean(samples, axis=1, dtype=np.float64)
    nmr_params, chain_length = samples.shape

    nmr_batches = int(np.floor(chain_length / batch_size))
    sigma = np.zeros((nmr_params, nmr_params))

    nmr_offsets = chain_length - nmr_batches * batch_size + 1

    for offset in range(nmr_offsets):
        batches = np.reshape(samples[:, np.array(offset + np.arange(0, nmr_batches * batch_size), dtype=np.int)].T,
                             [batch_size, nmr_batches, nmr_params], order='F')

        batch_means = np.squeeze(np.mean(batches, axis=0, dtype=np.float64))

        Z = batch_means - sample_means

        for x, y in itertools.product(range(nmr_params), range(nmr_params)):
            sigma[x, y] += np.sum(Z[:, x] * Z[:, y])

    return sigma * batch_size / (nmr_batches - 1) / nmr_offsets

Response: r Estimates the multivariate ESS sigma matrix for a set of samples.


Instruction:
Here you have a function in Python 3, explain what it does
def monte_carlo_standard_error(chain, batch_size_generator=None, compute_method=None):
    batch_size_generator = batch_size_generator or SquareRootSingleBatch()
    compute_method = compute_method or BatchMeansMCSE()

    batch_sizes = batch_size_generator.get_univariate_ess_batch_sizes(len(chain))

    return np.min(list(compute_method.compute_standard_error(chain, b) for b in batch_sizes))

Response:
Compute Monte Carlo standard errors for the expectations
 ArcGIS Markov chain.