Instruction:
Can you create a Python 3 function that
dispatches a Put request to the server to send the response to the server

Response:
def send_put(self, mri, attribute_name, value):
        """Abstract method to dispatch a Put to the server

        Args:
            mri (str): The mri of the Block
            attribute_name (str): The name of the Attribute within the Block
            value: The value to put
        """
        q = Queue()
        request = Put(
            path=[mri, attribute_name, "value"],
            value=value)
        request.set_callback(q.put)
        IOLoopHelper.call(self._send_request, request)
        response = q.get()
        if isinstance(response, Error):
            raise response.message
        else:
            return response.value


Instruction:
Explain what the following Python 3 code does
def send_post(self, mri, method_name, **params):
        q = Queue()
        request = Post(
            path=[mri, method_name],
            parameters=params)
        request.set_callback(q.put)
        IOLoopHelper.call(self._send_request, request)
        response = q.get()
        if isinstance(response, Error):
            raise response.message
        else:
            return response.value

Response: Dispatch a Post to the server


Instruction:
Given the following Python 3 function, write the documentation
def filter_parts(cls, part_info):
        # type: (Type[T], PartInfo) -> Dict[str, List[T]]
        """Filter the part_info dict looking for instances of our class

        Args:
            part_info (dict): {part_name: [Info] or None} as returned from
                Controller.run_hook()

        Returns:
            dict: {part_name: [info]} where info is a subclass of cls
        """
        filtered = OrderedDict()
        for part_name, info_list in part_info.items():
            if info_list is None or isinstance(info_list, Exception):
                continue
            info_list = [i for i in info_list if isinstance(i, cls)]
            if info_list:
                filtered[part_name] = info_list
        return filtered

Response: Filter the part_info dict looking for instances of cls in the class


Instruction:
Can you generate the documentation for the following Python 3 function
def filter_values(cls, part_info):
        # type: (Type[T], PartInfo) -> List[T]
        """Filter the part_info dict list looking for instances of our class

        Args:
            part_info (dict): {part_name: [Info] or None} as returned from
                Controller.run_hook()

        Returns:
            list: [info] where info is a subclass of cls
        """
        filtered = []
        for info_list in cls.filter_parts(part_info).values():
            filtered += info_list
        return filtered

Response: Filter the part_info dict list looking for instances of our class


Instruction:
How would you implement a function in Python 3 that
filters the part_info dict list looking for a single value of our class.

Response:
def filter_single_value(cls, part_info, error_msg=None):
        # type: (Type[T], PartInfo, str) -> T
        """Filter the part_info dict list looking for a single instance of our
        class

        Args:
            part_info (dict): {part_name: [Info] or None} as returned from
                Controller.run_hook()
            error_msg (str, optional): Specific error message to show if
                there isn't a single value

        Returns:
            info subclass of cls
        """
        filtered = cls.filter_values(part_info)
        if len(filtered) != 1:
            if error_msg is None:
                error_msg = "Expected a single %s, got %s of them" % \
                            (cls.__name__, len(filtered))
            raise BadValueError(error_msg)
        return filtered[0]


Instruction:
Write a Python 3 function for
disconnecting anyone listening to any of the given mris.

Response:
def disconnect_pv_clients(self, mris):
        # type: (List[str]) -> None
        """Disconnect anyone listening to any of the given mris"""
        for mri in mris:
            for pv in self._pvs.pop(mri, {}).values():
                # Close pv with force destroy on, this will call
                # onLastDisconnect
                pv.close(destroy=True, sync=True, timeout=1.0)


Instruction:
Create a Python 3 function to
add a Source Port tag to the tags list removing any other Source Ports

Response:
def with_source_port_tag(self, tags, connected_value):
        """Add a Source Port tag to the tags list, removing any other Source
        Ports"""
        new_tags = [t for t in tags if not t.startswith("sourcePort:")]
        new_tags.append(self.source_port_tag(connected_value))
        return new_tags


Instruction:
Can you write a function in Python 3 where it
searches tags for port info returning it if found

Response:
def port_tag_details(cls, tags):
        # type: (Sequence[str]) -> Union[Tuple[bool, Port, str], None]
        """Search tags for port info, returning it

        Args:
            tags: A list of tags to check

        Returns:
            None or (is_source, port, connected_value|disconnected_value)
            where port is one of the Enum entries of Port
        """
        for tag in tags:
            match = port_tag_re.match(tag)
            if match:
                source_sink, port, extra = match.groups()
                return source_sink == "source", cls(port), extra


Instruction:
Implement a Python 3 function for
transitioning the state of the current object to a new state.

Response:
def transition(self, state, message=""):
        """Change to a new state if the transition is allowed

        Args:
            state (str): State to transition to
            message (str): Message if the transition is to a fault state
        """
        with self.changes_squashed:
            initial_state = self.state.value
            if self.state_set.transition_allowed(
                    initial_state=initial_state, target_state=state):
                self.log.debug(
                    "%s: Transitioning from %s to %s",
                    self.mri, initial_state, state)
                if state == ss.DISABLED:
                    alarm = Alarm.invalid("Disabled")
                elif state == ss.FAULT:
                    alarm = Alarm.major(message)
                else:
                    alarm = Alarm()
                self.update_health(self, HealthInfo(alarm))
                self.state.set_value(state)
                self.state.set_alarm(alarm)
                for child, writeable in self._children_writeable[state].items():
                    if isinstance(child, AttributeModel):
                        child.meta.set_writeable(writeable)
                    elif isinstance(child, MethodModel):
                        child.set_writeable(writeable)
            else:
                raise TypeError("Cannot transition from %s to %s" %
                                (initial_state, state))


Instruction:
Given the following Python 3 function, write the documentation
def send_put(self, mri, attribute_name, value):
        path = attribute_name + ".value"
        typ, value = convert_to_type_tuple_value(serialize_object(value))
        if isinstance(typ, tuple):
            # Structure, make into a Value
            _, typeid, fields = typ
            value = Value(Type(fields, typeid), value)
        try:
            self._ctxt.put(mri, {path: value}, path)
        except RemoteError:
            if attribute_name == "exports":
                # TODO: use a tag instead of a name
                # This will change the structure of the block
                # Wait for reconnect
                self._queues[mri].get(timeout=DEFAULT_TIMEOUT)
            else:
                # Not expected, raise
                raise

Response: Dispatch a Put to the server


Instruction:
How would you explain what the following Python 3 function does
def send_post(self, mri, method_name, **params):
        typ, parameters = convert_to_type_tuple_value(serialize_object(params))
        uri = NTURI(typ[2])

        uri = uri.wrap(
            path="%s.%s" % (mri, method_name),
            kws=parameters,
            scheme="pva"
        )
        value = self._ctxt.rpc(mri, uri, timeout=None)
        return convert_value_to_dict(value)

Response: Abstract method to dispatch a Post to the server


Instruction:
Can you generate a brief explanation for the following Python 3 code
def make_view(self, context, data, child_name):
        # type: (Context, Model, str) -> Any
        """Make a child View of data[child_name]"""
        with self._lock:
            child = data[child_name]
            child_view = make_view(self, context, child)
        return child_view

Response: Make a child View of the child_name.


Instruction:
Can you generate the documentation for the following Python 3 function
def _handle_get(self, request):
        # type: (Get) -> CallbackResponses
        """Called with the lock taken"""
        data = self._block

        for i, endpoint in enumerate(request.path[1:]):
            try:
                data = data[endpoint]
            except KeyError:
                if hasattr(data, "typeid"):
                    typ = data.typeid
                else:
                    typ = type(data)
                raise UnexpectedError(
                    "Object %s of type %r has no attribute %r" % (
                        request.path[:i+1], typ, endpoint))
        # Important to serialize now with the lock so we get a consistent set
        serialized = serialize_object(data)
        ret = [request.return_response(serialized)]
        return ret

Response: Handles a GET request.


Instruction:
Write a Python 3 function that can
handle a Put request.

Response:
def _handle_put(self, request):
        # type: (Put) -> CallbackResponses
        """Called with the lock taken"""
        attribute_name = request.path[1]

        attribute = self._block[attribute_name]
        assert isinstance(attribute, AttributeModel), \
            "Cannot Put to %s which is a %s" % (attribute.path, type(attribute))
        self.check_field_writeable(attribute)

        put_function = self.get_put_function(attribute_name)
        value = attribute.meta.validate(request.value)

        with self.lock_released:
            result = put_function(value)

        if request.get and result is None:
            # We asked for a Get, and didn't get given a return, so do return
            # the current value. Don't serialize here as value is immutable
            # (as long as we don't try too hard to break the rules)
            result = self._block[attribute_name].value
        elif not request.get:
            # We didn't ask for a Get, so throw result away
            result = None
        ret = [request.return_response(result)]
        return ret


Instruction:
How would you implement a function in Python 3 that
calls with the lock taken

Response:
def _handle_post(self, request):
        # type: (Post) -> CallbackResponses
        """Called with the lock taken"""
        method_name = request.path[1]

        method = self._block[method_name]
        assert isinstance(method, MethodModel), \
            "Cannot Post to %s which is a %s" % (method.path, type(method))
        self.check_field_writeable(method)

        post_function = self.get_post_function(method_name)
        args = method.validate(request.parameters)

        with self.lock_released:
            result = post_function(**args)

        # Don't need to serialize as the result should be immutable
        ret = [request.return_response(result)]
        return ret


Instruction:
Implement a function in Python 3 to
wait until a StatefulController has initialized the child block

Response:
def wait_for_stateful_block_init(context, mri, timeout=DEFAULT_TIMEOUT):
    """Wait until a Block backed by a StatefulController has initialized

    Args:
        context (Context): The context to use to make the child block
        mri (str): The mri of the child block
        timeout (float): The maximum time to wait
    """
    context.when_matches(
        [mri, "state", "value"], StatefulStates.READY,
        bad_values=[StatefulStates.FAULT, StatefulStates.DISABLED],
        timeout=timeout)


Instruction:
Write a Python 3 script to
return the result of the call that the future represents.

Response:
def result(self, timeout=None):
        """Return the result of the call that the future represents.

        Args:
            timeout: The number of seconds to wait for the result if the future
                isn't done. If None, then there is no limit on the wait time.

        Returns:
            The result of the call that the future represents.

        Raises:
            TimeoutError: If the future didn't finish executing before the given
                timeout.
            exceptions.Exception: If the call raised then that exception will be
                raised.
        """
        if self._state == self.RUNNING:
            self._context.wait_all_futures([self], timeout)
        return self.__get_result()


Instruction:
Make a summary of the following Python 3 code
def exception(self, timeout=None):
        if self._state == self.RUNNING:
            self._context.wait_all_futures([self], timeout)
        return self._exception

Response: Returns the exception raised by the call that the future represents.


Instruction:
Write a Python 3 function for
setting the result of the work associated with the future.

Response:
def set_result(self, result):
        """Sets the return value of work associated with the future.

        Should only be used by Task and unit tests.
        """
        self._result = result
        self._state = self.FINISHED


Instruction:
Make a summary of the following Python 3 code
def set_exception(self, exception):
        assert isinstance(exception, Exception), \
            "%r should be an Exception" % exception
        self._exception = exception
        self._state = self.FINISHED

Response: Sets the result of the future as being the given exception.


Instruction:
Can you write a function in Python 3 where it
changes the name of the logger that log. should call

Response:
def set_logger(self, **fields):
        """Change the name of the logger that log.* should call

        Args:
            **fields: Extra fields to be logged. Logger name will be:
                ".".join([<module_name>, <cls_name>] + fields_sorted_on_key)
        """
        names = [self.__module__, self.__class__.__name__]
        for field, value in sorted(fields.items()):
            names.append(value)
        # names should be something like this for one field:
        #   ["malcolm.modules.scanning.controllers.runnablecontroller",
        #    "RunnableController", "BL45P-ML-SCAN-01"]
        self.log = logging.getLogger(".".join(names))
        if fields:
            self.log.addFilter(FieldFilter(fields))
        return self.log


Instruction:
How would you code a function in Python 3 to
create a Return Response object to signal a return value

Response:
def return_response(self, value=None):
        # type: (Any) -> Tuple[Callback, Return]
        """Create a Return Response object to signal a return value"""
        response = Return(id=self.id, value=value)
        return self.callback, response


Instruction:
Given the following Python 3 function, write the documentation
def error_response(self, exception):
        # type: (Exception) -> Tuple[Callback, Error]
        """Create an Error Response object to signal an error"""
        response = Error(id=self.id, message=exception)
        log.exception("Exception raised for request %s", self)
        return self.callback, response

Response: Create an Error Response object to signal an error


Instruction:
Create a Python 3 function for
making a View subclass containing properties specific for given data

Response:
def make_view(controller, context, data):
    # type: (Controller, Context, Any) -> Any
    """Make a View subclass containing properties specific for given data

    Args:
        controller (Controller): The child controller that hosts the data
        context (Context): The context the parent has made that the View should
            use for manipulating the data
        data (Model): The actual data that context will be manipulating

    Returns:
        View: A View subclass instance that provides a user-focused API to
            the given data
    """
    if isinstance(data, BlockModel):
        # Make an Block View
        view = _make_view_subclass(Block, controller, context, data)
    elif isinstance(data, AttributeModel):
        # Make an Attribute View
        view = Attribute(controller, context, data)
    elif isinstance(data, MethodModel):
        # Make a Method View
        view = Method(controller, context, data)
    elif isinstance(data, Model):
        # Make a generic View
        view = _make_view_subclass(View, controller, context, data)
    elif isinstance(data, dict):
        # Make a dict of Views
        d = OrderedDict()
        for k, v in data.items():
            d[k] = make_view(controller, context, v)
        view = d
    elif isinstance(data, list):
        # Need to recurse down
        view = [make_view(controller, context, x) for x in data]
    else:
        # Just return the data unwrapped as it should be immutable
        view = data
    return view


Instruction:
How would you code a function in Python 3 to
put a value to the Attribute and wait for completion

Response:
def put_value(self, value, timeout=None):
        """Put a value to the Attribute and wait for completion"""
        self._context.put(self._data.path + ["value"], value, timeout=timeout)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def calculate_exposure(self, duration):
        # type: (float) -> float
        """Calculate the exposure to set the detector to given the duration of
        the frame and the readout_time and frequency_accuracy"""
        exposure = duration - self.frequency_accuracy * duration / 1000000.0 - \
                   self.readout_time
        assert exposure > 0.0, \
            "Exposure time %s too small when deadtime taken into account" % (
                exposure,)
        return exposure

Response:
Calculate the exposure to set the detector to given the duration of
        the frame and the readout_time and frequency_accuracy.


Instruction:
Create a Python 3 function for
setting the notifier and the path from the path from the block root

Response:
def set_notifier_path(self, notifier, path):
        """Sets the notifier, and the path from the path from block root

        Args:
            notifier (Notifier): The Notifier to tell when endpoint data changes
            path (list): The absolute path to get to this object
        """
        # type: (Union[Notifier, DummyNotifier], List[str]) -> None
        # This function should either change from the DummyNotifier or to
        # the DummyNotifier, never between two valid notifiers
        assert self.notifier is Model.notifier or notifier is Model.notifier, \
            "Already have a notifier %s path %s" % (self.notifier, self.path)
        self.notifier = notifier
        self.path = path
        # Tell all our children too
        for name, ct in self.call_types.items():
            if ct.is_mapping:
                child = getattr(self, name)
                if child and issubclass(ct.typ[1], Model):
                    for k, v in child.items():
                        v.set_notifier_path(notifier, self.path + [name, k])
            elif issubclass(ct.typ, Model):
                assert not ct.is_array, \
                    "Can't deal with Arrays of Models %s" % ct
                child = getattr(self, name)
                child.set_notifier_path(notifier, self.path + [name])


Instruction:
Can you generate a brief explanation for the following Python 3 code
def apply_change(self, path, *args):
        # type: (List[str], Any) -> None
        """Take a single change from a Delta and apply it to this model"""
        if len(path) > 1:
            # This is for a child
            self[path[0]].apply_change(path[1:], *args)
        else:
            # This is for us
            assert len(path) == 1 and len(args) == 1, \
                "Cannot process change %s" % ([self.path + path] + list(args))
            getattr(self, "set_%s" % path[0])(args[0])

Response: Take a single change from a Delta and apply it to this model


Instruction:
Can you implement a function in Python 3 that
creates an AttributeModel instance of the correct type for this Meta

Response:
def create_attribute_model(self, initial_value=None):
        # type: (Any) -> AttributeModel
        """Make an AttributeModel instance of the correct type for this Meta

        Args:
            initial_value: The initial value the Attribute should take

        Returns:
            AttributeModel: The created attribute model instance
        """
        attr = self.attribute_class(meta=self, value=initial_value)
        return attr


Instruction:
Write a Python 3 script to
return an instance of this class from an Anno.

Response:
def from_annotype(cls, anno, writeable, **kwargs):
        # type: (Anno, bool, **Any) -> VMeta
        """Return an instance of this class from an Anno"""
        ret = cls(description=anno.description, writeable=writeable, **kwargs)
        widget = ret.default_widget()
        if widget != Widget.NONE:
            ret.set_tags([widget.tag()])
        return ret


Instruction:
Can you generate the documentation for the following Python 3 function
def register_annotype_converter(cls, types, is_array=False,
                                    is_mapping=False):
        # type: (Union[Sequence[type], type], bool, bool) -> Any
        """Register this class as a converter for Anno instances"""
        if not isinstance(types, Sequence):
            types = [types]

        def decorator(subclass):
            for typ in types:
                cls._annotype_lookup[(typ, is_array, is_mapping)] = subclass
            return subclass

        return decorator

Response: Register this class as a converter for Anno instances


Instruction:
Implement a function in Python 3 to
look up a vmeta based on an Anno

Response:
def lookup_annotype_converter(cls, anno):
        # type: (Anno) -> Type[VMeta]
        """Look up a vmeta based on an Anno"""
        if hasattr(anno.typ, "__bases__"):
            # This is a proper type
            bases = inspect.getmro(anno.typ)
        else:
            # This is a numpy dtype
            bases = [anno.typ]
        for typ in bases:
            key = (typ, bool(anno.is_array), bool(anno.is_mapping))
            try:
                return cls._annotype_lookup[key]
            except KeyError:
                pass
        raise KeyError(anno)


Instruction:
Here you have a function in Python 3, explain what it does
def set_value(self, value, set_alarm_ts=True, alarm=None, ts=None):
        # type: (Any, bool, Alarm, TimeStamp) -> Any
        """Set value, calculating alarm and ts if requested"""
        value = self.meta.validate(value)
        if set_alarm_ts:
            if alarm is None:
                alarm = Alarm.ok
            else:
                alarm = deserialize_object(alarm, Alarm)
            if ts is None:
                ts = TimeStamp()
            else:
                ts = deserialize_object(ts, TimeStamp)
            self.set_value_alarm_ts(value, alarm, ts)
        else:
            self.set_endpoint_data("value", value)
        return self.value

Response: Set value for the current instance of the class.


Instruction:
How would you code a function in Python 3 to
set value with pre - validated alarm and timeStamp.

Response:
def set_value_alarm_ts(self, value, alarm, ts):
        """Set value with pre-validated alarm and timeStamp"""
        # type: (Any, Alarm, TimeStamp) -> None
        with self.notifier.changes_squashed:
            # Assume they are of the right format
            self.value = value
            self.notifier.add_squashed_change(self.path + ["value"], value)
            if alarm is not self.alarm:
                self.alarm = alarm
                self.notifier.add_squashed_change(self.path + ["alarm"], alarm)
            self.timeStamp = ts
            self.notifier.add_squashed_change(self.path + ["timeStamp"], ts)


Instruction:
Write a Python 3 function that can
return a greeting to name.

Response:
def greet(self, name, sleep=0):
        # type: (AName, ASleep) -> AGreeting
        """Optionally sleep <sleep> seconds, then return a greeting to <name>"""
        print("Manufacturing greeting...")
        sleep_for(sleep)
        greeting = "Hello %s" % name
        return greeting


Instruction:
Implement a function in Python 3 to
extract the version number from the version. py file.

Response:
def get_version():
    """Extracts the version number from the version.py file."""
    VERSION_FILE = '../malcolm/version.py'
    mo = re.search(r'^__version__ = [\'"]([^\'"]*)[\'"]',
                   open(VERSION_FILE, 'rt').read(), re.M)
    if mo:
        return mo.group(1)
    else:
        raise RuntimeError(
            'Unable to find version string in {0}.'.format(VERSION_FILE))


Instruction:
Explain what the following Python 3 code does
def add_nexus_nodes(generator, vds_file_path):

    # create the axes dimensions attribute, a comma separated list giving size
    # of the axis dimensions padded with . for the detector dimensions and
    # multidimensional dimensions
    pad_dims = []
    for d in generator.dimensions:
        if len(d.axes) == 1:
            pad_dims.append("%s_set" % d.axes[0])
        else:
            pad_dims.append(".")

    pad_dims += ["."] * 2  # assume a 2 dimensional detector

    with h5py.File(vds_file_path, 'r+', libver="latest") as vds:
        for data, node in zip(set_data, set_bases):
            # create a group for this entry
            vds.require_group(node)
            # points to the axis demand data sets
            vds[node].attrs["axes"] = pad_dims
            vds[node].attrs["NX_class"] = ['NXdata']

            # points to the detector dataset for this entry
            vds[node].attrs["signal"] = data.split('/')[-1]
            # a hard link from this entry 'signal' to the actual data
            vds[node + data] = vds[data]

            axis_sets = {}
            # iterate the axes in each dimension of the generator to create the
            # axis information nodes
            for i, d in enumerate(generator.dimensions):
                for axis in d.axes:
                    # add signal data dimension for axis
                    axis_indices = '{}_set_indices'.format(axis)
                    vds[node].attrs[axis_indices] = i

                    # demand positions for axis
                    axis_set = '{}_set'.format(axis)
                    if axis_sets.get(axis_set):
                        # link to the first entry's demand list
                        vds[node + axis_set] = axis_sets[axis_set]
                    else:
                        # create the demand list for the first entry only
                        axis_demands = d.get_positions(axis)
                        vds.create_dataset(
                            node + axis_set, data=axis_demands)
                        vds[node + axis_set].attrs["units"] = \
                            generator.units[axis]
                    axis_sets[axis_set] = vds[node + axis_set]

        vds['entry'].attrs["NX_class"] = ['NXentry']

Response: Add in the additional information to make this into a standard nexus tree.


Instruction:
Explain what the following Python 3 code does
def send_recv(self, message, timeout=10.0):
        response_queue = self.send(message)
        response = self.recv(response_queue, timeout)
        return response

Response: Send a message to a PandABox and wait for the response


Instruction:
Given the following Python 3 function, write the documentation
def _send_loop(self):
        while True:
            message, response_queue = self._send_queue.get()
            if message is self.STOP:
                break
            try:
                self._response_queues.put(response_queue)
                self._socket.send(message)
            except Exception:  # pylint:disable=broad-except
                log.exception("Exception sending message %s", message)

Response: Send a message to the server.


Instruction:
Can you implement a function in Python 3 that
responds to the person waiting

Response:
def _respond(self, resp):
        """Respond to the person waiting"""
        response_queue = self._response_queues.get(timeout=0.1)
        response_queue.put(resp)
        self._completed_response_lines = []
        self._is_multiline = None


Instruction:
How would you explain what the following Python 3 function does
def _recv_loop(self):
        self._completed_response_lines = []
        self._is_multiline = None
        lines_iterator = self._get_lines()
        while True:
            try:
                line = next(lines_iterator)
                if self._is_multiline is None:
                    self._is_multiline = line.startswith("!") or line == "."
                if line.startswith("ERR"):
                    self._respond(ValueError(line))
                elif self._is_multiline:
                    if line == ".":
                        self._respond(self._completed_response_lines)
                    else:
                        assert line[0] == "!", \
                            "Multiline response {} doesn't start with !" \
                            .format(repr(line))
                        self._completed_response_lines.append(line[1:])
                else:
                    self._respond(line)
            except StopIteration:
                return
            except Exception:
                log.exception("Exception receiving message")
                raise

Response: This method is called by the service socket to receive messages from the server.


Instruction:
Can you tell what is the following Python 3 function doing
def parameterized_send(self, request, parameter_list):
        response_queues = OrderedDict()
        for parameter in parameter_list:
            response_queues[parameter] = self.send(request % parameter)
        return response_queues

Response: Send a request to the server with a list of parameters


Instruction:
Can you write a function in Python 3 where it
calculates the readout time of the detector from the EPICS driver

Response:
def get_readout_time(self, child, duration):
        """Calculate the readout time of the detector from the EPICS driver:
            - Set exposure and acquire period to same value
            - Acquire period will be set to lowest acceptable value
            - Difference will be readout time (this value is affected by
              detector settings)
        """
        child.exposure.put_value(duration)
        child.acquirePeriod.put_value(duration)
        readout_time = child.acquirePeriod.value - child.exposure.value
        # It seems that the difference between acquirePeriod and exposure
        # doesn't tell the whole story, we seem to need an additional bit
        # of readout (or something) time on top
        fudge_factor = duration * 0.004 + 0.001
        return readout_time + fudge_factor


Instruction:
How would you explain what the following Python 3 function does
def notify_dispatch_request(self, request):
        # type: (Request) -> None
        """Will be called when a context passed to a hooked function is about
        to dispatch a request"""
        if isinstance(request, Put) and request.path[0] == self.mri:
            # This means the context we were passed has just made a Put request
            # so mark the field as "we_modified" so it doesn't screw up the
            # modified led
            attribute_name = request.path[-2]
            if attribute_name not in self.no_save:
                self.log.warning(
                    "Part %s tried to set '%s' that is not in self.no_save. "
                    "This will stop the 'modified' attribute from working.",
                    self, attribute_name)

Response: Called by the hooked function when a request is about to dispatch a new object.


Instruction:
Can you generate the documentation for the following Python 3 function
def sever_sink_ports(self, context, ports, connected_to=None):
        # type: (AContext, APortMap, str) -> None
        """Conditionally sever Sink Ports of the child. If connected_to
        is then None then sever all, otherwise restrict to connected_to's
        Source Ports

        Args:
            context (Context): The context to use
            ports (dict): {part_name: [PortInfo]}
            connected_to (str): Restrict severing to this part
        """
        # Find the Source Ports to connect to
        if connected_to:
            # Calculate a lookup of the Source Port "name" to type
            source_port_lookup = self._source_port_lookup(
                ports.get(connected_to, []))
        else:
            source_port_lookup = True

        # Find our Sink Ports
        sink_ports = self._get_flowgraph_ports(ports, SinkPortInfo)

        # If we have Sunk Ports that need to be disconnected then do so
        if sink_ports and source_port_lookup:
            child = context.block_view(self.mri)
            attribute_values = {}
            for name, port_info in sink_ports.items():
                if source_port_lookup is True or source_port_lookup.get(
                        child[name].value, None) == port_info.port:
                    attribute_values[name] = port_info.disconnected_value
            child.put_attribute_values(attribute_values)

Response: Conditionally sever Sink Ports of the child.


Instruction:
Can you write a function in Python 3 where it
calculates what is connected to what

Response:
def calculate_part_visibility(self, ports):
        # type: (APortMap) -> None
        """Calculate what is connected to what

        Args:
            ports: {part_name: [PortInfo]} from other ports
        """
        # Calculate a lookup of Source Port connected_value to part_name
        source_port_lookup = {}
        for part_name, port_infos in SourcePortInfo.filter_parts(ports).items():
            for port_info in port_infos:
                source_port_lookup[port_info.connected_value] = (
                    part_name, port_info.port)
        # Look through all the Sink Ports, and set both ends of the
        # connection to visible if they aren't specified
        for part_name, port_infos in SinkPortInfo.filter_parts(
                ports).items():
            for port_info in port_infos:
                if port_info.value != port_info.disconnected_value:
                    conn_part, port = source_port_lookup.get(
                        port_info.value, (None, None))
                    if conn_part and port == port_info.port:
                        if conn_part not in self.part_visibility:
                            self.part_visibility[conn_part] = True
                        if part_name not in self.part_visibility:
                            self.part_visibility[part_name] = True


Instruction:
Here you have a function in Python 3, explain what it does
def handle_subscribe(self, request):
        # type: (Subscribe) -> CallbackResponses
        """Handle a Subscribe request from outside. Called with lock taken"""
        ret = self._tree.handle_subscribe(request, request.path[1:])
        self._subscription_keys[request.generate_key()] = request
        return ret

Response: Handle a Subscribe request from outside. Called with lock taken.


Instruction:
Write a Python 3 function that can
handle a Unsubscribe request from outside. Called with lock taken.

Response:
def handle_unsubscribe(self, request):
        # type: (Unsubscribe) -> CallbackResponses
        """Handle a Unsubscribe request from outside. Called with lock taken"""
        subscribe = self._subscription_keys.pop(request.generate_key())
        ret = self._tree.handle_unsubscribe(subscribe, subscribe.path[1:])
        return ret


Instruction:
Can you implement a function in Python 3 that
registers a squashed change to a particular path

Response:
def add_squashed_change(self, path, data):
        # type: (List[str], Any) -> None
        """Register a squashed change to a particular path

        Args:
            path (list): The path of what has changed, relative from Block
            data (object): The new data
        """
        assert self._squashed_count, "Called while not squashing changes"
        self._squashed_changes.append([path[1:], data])


Instruction:
Can you implement a function in Python 3 that
sets our data and notify anyone listening to the changes

Response:
def notify_changes(self, changes):
        # type: (List[List]) -> CallbackResponses
        """Set our data and notify anyone listening

        Args:
            changes (list): [[path, optional data]] where path is the path to
                what has changed, and data is the unserialized object that has
                changed

        Returns:
            list: [(callback, Response)] that need to be called
        """
        ret = []
        child_changes = {}
        for change in changes:
            # Add any changes that our children need to know about
            self._add_child_change(change, child_changes)

        # If we have update subscribers, serialize at this level
        if self.update_requests:
            serialized = serialize_object(self.data)
            for request in self.update_requests:
                ret.append(request.update_response(serialized))

        # If we have delta subscribers, serialize the changes
        if self.delta_requests:
            for change in changes:
                change[-1] = serialize_object(change[-1])
            for request in self.delta_requests:
                ret.append(request.delta_response(changes))

        # Now notify our children
        for name, child_changes in child_changes.items():
            ret += self.children[name].notify_changes(child_changes)
        return ret


Instruction:
Write a Python 3 function for
updating our data and notify any subscribers of children what has changed

Response:
def _update_data(self, data):
        # type: (Any) -> Dict[str, List]
        """Set our data and notify any subscribers of children what has changed

        Args:
            data (object): The new data

        Returns:
            dict: {child_name: [path_list, optional child_data]} of the change
                that needs to be passed to a child as a result of this
        """
        self.data = data
        child_change_dict = {}
        # Reflect change of data to children
        for name in self.children:
            child_data = getattr(data, name, None)
            if child_data is None:
                # Deletion
                child_change_dict[name] = [[]]
            else:
                # Change
                child_change_dict[name] = [[], child_data]
        return child_change_dict


Instruction:
Create a Python 3 function for
adding to the list of request to notify and notify the initial value of

Response:
def handle_subscribe(self, request, path):
        # type: (Subscribe, List[str]) -> CallbackResponses
        """Add to the list of request to notify, and notify the initial value of
        the data held

        Args:
            request (Subscribe): The subscribe request
            path (list): The relative path from ourself

        Returns:
            list: [(callback, Response)] that need to be called
        """
        ret = []
        if path:
            # Recurse down
            name = path[0]
            if name not in self.children:
                self.children[name] = NotifierNode(
                    getattr(self.data, name, None), self)
            ret += self.children[name].handle_subscribe(request, path[1:])
        else:
            # This is for us
            serialized = serialize_object(self.data)
            if request.delta:
                self.delta_requests.append(request)
                ret.append(request.delta_response([[[], serialized]]))
            else:
                self.update_requests.append(request)
                ret.append(request.update_response(serialized))
        return ret


Instruction:
Explain what the following Python 3 code does
def handle_unsubscribe(self, request, path):
        # type: (Subscribe, List[str]) -> CallbackResponses
        """Remove from the notifier list and send a return

        Args:
            request (Subscribe): The original subscribe request
            path (list): The relative path from ourself

        Returns:
            list: [(callback, Response)] that need to be called
        """
        ret = []
        if path:
            # Recurse down
            name = path[0]
            child = self.children[name]
            ret += child.handle_unsubscribe(request, path[1:])
            if not child.children and not child.update_requests \
                    and not child.delta_requests:
                del self.children[name]
        else:
            # This is for us
            if request in self.update_requests:
                self.update_requests.remove(request)
            else:
                self.delta_requests.remove(request)
            ret.append(request.return_response())
        return ret

Response: This function is called by the user when a subscription is removed from the notifier list and the return_response is sent to the user when the user is done with it.


Instruction:
Write a Python 3 script to
add a string parameter to be passed when instantiating this YAML file

Response:
def string(name, description, default=None):
    # type: (AName, ADescription, AStringDefault) -> AAnno
    """Add a string parameter to be passed when instantiating this YAML file"""
    args = common_args(name, default)
    return Anno(description, typ=str, **args)


Instruction:
Write a Python 3 script for
adding a float64 parameter to be passed when instantiating this YAML file

Response:
def float64(name, description, default=None):
    # type: (AName, ADescription, AFloat64Default) -> AAnno
    """Add a float64 parameter to be passed when instantiating this YAML file"""
    args = common_args(name, default)
    return Anno(description, typ=float, **args)


Instruction:
Can you generate the documentation for the following Python 3 function
def int32(name, description, default=None):
    # type: (AName, ADescription, AInt32Default) -> AAnno
    """Add an int32 parameter to be passed when instantiating this YAML file"""
    args = common_args(name, default)
    return Anno(description, typ=int, **args)

Response: Add an int32 parameter to be passed when instantiating this YAML file


Instruction:
Given the following Python 3 function, write the documentation
def make_block_creator(yaml_path, filename=None):
    # type: (str, str) -> Callable[..., List[Controller]]
    """Make a collection function that will create a list of blocks

    Args:
        yaml_path (str): File path to YAML file, or a file in the same dir
        filename (str): If give, use this filename as the last element in
            the yaml_path (so yaml_path can be __file__)

    Returns:
        function: A collection function decorated with @takes. This can be
            used in other blocks or instantiated by the process. If the
            YAML text specified controllers or parts then a block instance
            with the given name will be instantiated. If there are any
            blocks listed then they will be called. All created blocks
            by this or any sub collection will be returned
    """
    sections, yamlname, docstring = Section.from_yaml(yaml_path, filename)
    yamldir = os.path.dirname(yaml_path)

    # Check we have only one controller
    controller_sections = [s for s in sections if s.section == "controllers"]
    assert len(controller_sections) == 1, \
        "Expected exactly 1 controller, got %s" % (controller_sections,)
    controller_section = controller_sections[0]

    def block_creator(kwargs):
        # Create the param dict of the static defined arguments
        defines = _create_defines(sections, yamlname, yamldir, kwargs)
        controllers, parts = _create_blocks_and_parts(sections, defines)
        # Make the controller
        controller = controller_section.instantiate(defines)
        for part in parts:
            controller.add_part(part)
        controllers.append(controller)
        return controllers

    creator = creator_with_nice_signature(
        block_creator, sections, yamlname, yaml_path, docstring)
    return creator

Response: Create a function that creates a list of blocks with the given name.


Instruction:
Write a Python 3 function for
keeping recursing down from base using dotted name then call it with self. params and args

Response:
def instantiate(self, substitutions):
        """Keep recursing down from base using dotted name, then call it with
        self.params and args

        Args:
            substitutions (dict): Substitutions to make to self.param_dict

        Returns:
            The found object called with (*args, map_from_d)

        E.g. if ob is malcolm.parts, and name is "ca.CADoublePart", then the
        object will be malcolm.parts.ca.CADoublePart
        """
        param_dict = self.substitute_params(substitutions)
        pkg, ident = self.name.rsplit(".", 1)
        pkg = "malcolm.modules.%s" % pkg
        try:
            ob = importlib.import_module(pkg)
        except ImportError as e:
            raise_with_traceback(
                ImportError("\n%s:%d:\n%s" % (
                    self.filename, self.lineno, e)))
        try:
            ob = getattr(ob, ident)
        except AttributeError:
            raise_with_traceback(
                ImportError("\n%s:%d:\nPackage %r has no ident %r" % (
                    self.filename, self.lineno, pkg, ident)))
        try:
            model = MethodModel.from_callable(ob, returns=False)
            args = model.validate(param_dict)
            ret = ob(**args)
        except Exception as e:
            sourcefile = inspect.getsourcefile(ob)
            lineno = inspect.getsourcelines(ob)[1]
            raise_with_traceback(
                YamlError("\n%s:%d:\n%s:%d:\n%s" % (
                    self.filename, self.lineno, sourcefile, lineno, e)))
        else:
            return ret


Instruction:
Implement a function in Python 3 to
parse a YAML file and return a tuple of the sections yamlname and docstring

Response:
def from_yaml(cls, yaml_path, filename=None):
        """Split a dictionary into parameters controllers parts blocks defines

        Args:
            yaml_path (str): File path to YAML file, or a file in the same dir
            filename (str): If give, use this filename as the last element in
                the yaml_path (so yaml_path can be __file__)

        Returns:
            tuple: (sections, yamlname, docstring) where sections is a
                list of created sections
        """
        if filename:
            # different filename to support passing __file__
            yaml_path = os.path.join(os.path.dirname(yaml_path), filename)
        assert yaml_path.endswith(".yaml"), \
            "Expected a/path/to/<yamlname>.yaml, got %r" % yaml_path
        yamlname = os.path.basename(yaml_path)[:-5]
        log.debug("Parsing %s", yaml_path)
        with open(yaml_path) as f:
            text = f.read()
        # First separate them into their relevant sections
        ds = yaml.load(text, Loader=yaml.RoundTripLoader)
        docstring = None
        sections = []
        for d in ds:
            assert len(d) == 1, \
                "Expected section length 1, got %d" % len(d)
            lineno = d._yaml_line_col.line + 1
            name = list(d)[0]
            sections.append(cls(
                yaml_path, lineno, name, d[name]))
            if name == "builtin.defines.docstring":
                docstring = d[name]["value"]

        return sections, yamlname, docstring


Instruction:
Write a Python 3 script to
substitute param values in our param_dict from params

Response:
def substitute_params(self, substitutions):
        """Substitute param values in our param_dict from params

        Args:
            substitutions (Map or dict): Values to substitute. E.g. Map of
                {"name": "me"}

        E.g. if self.param_dict is:
            {"name": "$(name):pos", "exposure": 1.0}
        And substitutions is:
            {"name": "me"}
        After the call self.param_dict will be:
            {"name": "me:pos", "exposure": 1.0}
        """
        param_dict = {}
        # TODO: this should be yaml.add_implicit_resolver()
        for k, v in self.param_dict.items():
            param_dict[k] = replace_substitutions(v, substitutions)
        return param_dict


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _make_padded_ramp(self, v1, v2, pad_velocity, total_time):
        # The time taken to ramp from v1 to pad_velocity
        t1 = self.acceleration_time(v1, pad_velocity)
        # Then on to v2
        t2 = self.acceleration_time(pad_velocity, v2)
        # The distance during the pad
        tp = total_time - t1 - t2
        # Yield the points
        yield t1, pad_velocity
        yield tp, pad_velocity
        yield t2, v2

Response: Makes a ramp that looks like this : v1 \ __ pad_velocity \ __ v2 \ __ pad_velocity \ __ total_time \ __


Instruction:
Create a Python 3 function for
making a hat that looks like this : _ vm v1 v2 | \ d1 \ dm \ d2 \ d3 \ min_time \ _ vm \ min_time \ _ vm \ min_time \ _ vm \ min_time \ _ vm \ min_time \ _ vm \ min_time \ _ vm \ min_time \ _ vm \ min_time \ _ vm \ min_time \ _ vm \ min_time \ min_time \ min_time

Response:
def _make_hat(self, v1, v2, acceleration, distance, min_time):
        """Make a hat that looks like this:

            ______ vm
        v1 /|   | \
          d1| dm|d2\ v2
            |   |
          t1  tm t2

        Such that the area under the graph (d1+d2+d3) is distance and
        t1+t2+t3 >= min_time
        """
        if min_time > 0:
            # We are trying to meet time constraints
            # Solve quadratic to give vm
            b = v1 + v2 + min_time * acceleration
            c = distance * acceleration + (v1*v1 + v2*v2) / 2
            op = b*b - 4 * c
            if np.isclose(op, 0):
                # Might have a negative number as rounding error...
                op = 0
            elif op < 0:
                # Can't do this, set something massive to fail vm check...
                op = 10000000000

            def get_times(vm):
                t1 = (vm - v1) / acceleration
                t2 = (vm - v2) / acceleration
                tm = min_time - t1 - t2
                assert -self.max_velocity <= vm <= self.max_velocity
                assert t1 >= 0 and t2 >= 0 and tm >= 0
                return t1, tm, t2

            try:
                # Try negative root
                vm = (b - np.sqrt(op)) / 2
                t1, tm, t2 = get_times(vm)
            except AssertionError:
                try:
                    # Try positive root
                    vm = (b + np.sqrt(op)) / 2
                    t1, tm, t2 = get_times(vm)
                except AssertionError:
                    # If vm is out of range or any segment takes negative time,
                    # we can't do it in min_time, so act as if unconstrained
                    t1, tm, t2, vm = self._calculate_hat_params(
                        v1, v2, acceleration, distance)
        else:
            t1, tm, t2, vm = self._calculate_hat_params(
                v1, v2, acceleration, distance)

        # If middle segment needs to be negative time then we need to cap
        # vm and spend no time at vm
        if tm < 0:
            # Solve the quadratic to work out how long to spend accelerating
            vm = np.sqrt(
                (2 * acceleration * distance + v1 * v1 + v2 * v2) / 2)
            if acceleration < 0:
                vm = -vm
            t1 = self.acceleration_time(v1, vm)
            t2 = self.acceleration_time(vm, v2)
            tm = 0

        # Yield the result
        yield t1, vm
        yield tm, vm
        yield t2, v2


Instruction:
Write a Python 3 script for
calculating PVT points that will perform the move within motor params Args: v1 (float): Starting velocity in EGUs/s v2 (float): Ending velocity in EGUs/s distance (float): Relative distance to travel in EGUs min_time (float): The minimum time the move should take Returns: tuple: (time_list, position_list) where time_list is a list of relative time points in seconds, and position_list is the position in EGUs that the motor should be

Response:
def make_velocity_profile(self, v1, v2, distance, min_time):
        """Calculate PVT points that will perform the move within motor params

        Args:
            v1 (float): Starting velocity in EGUs/s
            v2 (float): Ending velocity in EGUs/s
            distance (float): Relative distance to travel in EGUs
            min_time (float): The minimum time the move should take

        Returns:
            tuple: (time_list, position_list) where time_list is a list of
                relative time points in seconds, and position_list is the
                position in EGUs that the motor should be
        """
        # Take off the settle time and distance
        if min_time > 0:
            min_time -= self.velocity_settle
        distance -= self.velocity_settle * v2
        # The ramp time and distance of a continuous ramp from v1 to v2
        ramp_time = self.acceleration_time(v1, v2)
        ramp_distance = self.ramp_distance(v1, v2, ramp_time)
        remaining_distance = distance - ramp_distance
        # Check if we need to stretch in time
        if min_time > ramp_time:
            # Check how fast we would need to be going so that the total move
            # completes in min_time
            pad_velocity = remaining_distance / (min_time - ramp_time)
            if pad_velocity > max(v1, v2):
                # Can't just pad the ramp, make a hat pointing up
                it = self._make_hat(
                    v1, v2, self.acceleration, distance, min_time)
            elif pad_velocity < min(v1, v2):
                # Can't just pad the ramp, make a hat pointing down
                it = self._make_hat(
                    v1, v2, -self.acceleration, distance, min_time)
            else:
                # Make a padded ramp
                it = self._make_padded_ramp(v1, v2, pad_velocity, min_time)
        elif remaining_distance < 0:
            # Make a hat pointing down
            it = self._make_hat(v1, v2, -self.acceleration, distance, min_time)
        else:
            # Make a hat pointing up
            it = self._make_hat(v1, v2, self.acceleration, distance, min_time)
        # Create the time and velocity arrays
        time_array = [0.0]
        velocity_array = [v1]
        for t, v in it:
            assert t >= 0, "Got negative t %s" % t
            if t == 0:
                assert v == velocity_array[-1], \
                    "Can't move velocity in zero time"
                continue
            if v * velocity_array[-1] < 0:
                # Crossed zero, put in an explicit zero velocity
                fraction = velocity_array[-1] / (velocity_array[-1] - v)
                time_array.append(time_array[-1] + fraction * t)
                velocity_array.append(0)
                t -= fraction * t
            time_array.append(time_array[-1] + t)
            velocity_array.append(v)
        # Add on the settle time
        if self.velocity_settle > 0:
            time_array.append(time_array[-1] + self.velocity_settle)
            velocity_array.append(v2)
        return time_array, velocity_array


Instruction:
Can you create a Python 3 function that
gives the parts of a CS and a set of axes to move return the cs_port and the mapping of cs_axis to MotorInfo.

Response:
def cs_axis_mapping(cls,
                        part_info,  # type: Dict[str, Optional[Sequence]]
                        axes_to_move  # type: Sequence[str]
                        ):
        # type: (...) -> Tuple[str, Dict[str, MotorInfo]]
        """Given the motor infos for the parts, filter those with scannable
        names in axes_to_move, check they are all in the same CS, and return
        the cs_port and mapping of cs_axis to MotorInfo"""
        cs_ports = set()  # type: Set[str]
        axis_mapping = {}  # type: Dict[str, MotorInfo]
        for motor_info in cls.filter_values(part_info):
            if motor_info.scannable in axes_to_move:
                assert motor_info.cs_axis in cs_axis_names, \
                    "Can only scan 1-1 mappings, %r is %r" % \
                    (motor_info.scannable, motor_info.cs_axis)
                cs_ports.add(motor_info.cs_port)
                axis_mapping[motor_info.scannable] = motor_info
        missing = list(set(axes_to_move) - set(axis_mapping))
        assert not missing, \
            "Some scannables %s are not in the CS mapping %s" % (
                missing, axis_mapping)
        assert len(cs_ports) == 1, \
            "Requested axes %s are in multiple CS numbers %s" % (
                axes_to_move, list(cs_ports))
        cs_axis_counts = Counter([x.cs_axis for x in axis_mapping.values()])
        # Any cs_axis defs that are used for more that one raw motor
        overlap = [k for k, v in cs_axis_counts.items() if v > 1]
        assert not overlap, \
            "CS axis defs %s have more that one raw motor attached" % overlap
        return cs_ports.pop(), axis_mapping


Instruction:
Create a Python 3 function to
set the layout table value. Called on attribute put.

Response:
def set_layout(self, value):
        """Set the layout table value. Called on attribute put"""
        # Can't do this with changes_squashed as it will call update_modified
        # from another thread and deadlock. Need RLock.is_owned() from update_*
        part_info = self.run_hooks(
            LayoutHook(p, c, self.port_info, value)
            for p, c in self.create_part_contexts(only_visible=False).items())
        with self.changes_squashed:
            layout_parts = LayoutInfo.filter_parts(part_info)
            name, mri, x, y, visible = [], [], [], [], []
            for part_name, layout_infos in layout_parts.items():
                for layout_info in layout_infos:
                    name.append(part_name)
                    mri.append(layout_info.mri)
                    x.append(layout_info.x)
                    y.append(layout_info.y)
                    visible.append(layout_info.visible)
            layout_table = LayoutTable(name, mri, x, y, visible)
            try:
                # Compare the Array seq to get at the numpy array
                np.testing.assert_equal(
                    layout_table.visible.seq, self.layout.value.visible.seq)
            except AssertionError:
                visibility_changed = True
            else:
                visibility_changed = False
            self.layout.set_value(layout_table)
            if self.saved_visibility is None:
                # First write of table, set layout and exports saves
                self.saved_visibility = layout_table.visible
                self.saved_exports = self.exports.value.to_dict()
                # Force visibility changed so we update_block_endpoints
                # even if there weren't any visible
                visibility_changed = True
            if visibility_changed:
                self.update_modified()
                self.update_exportable()
                # Part visibility changed, might have attributes or methods
                # that we need to hide or show
                self.update_block_endpoints()


Instruction:
How would you implement a function in Python 3 that
saves the current design to file

Response:
def save(self, designName=""):
        # type: (ASaveDesign) -> None
        """Save the current design to file"""
        self.try_stateful_function(
            ss.SAVING, ss.READY, self.do_save, designName)


Instruction:
How would you code a function in Python 3 to
make config dir and return full file path and extension

Response:
def _validated_config_filename(self, name):
        """Make config dir and return full file path and extension

        Args:
            name (str): Filename without dir or extension

        Returns:
            str: Full path including extension
        """
        dir_name = self._make_config_dir()
        filename = os.path.join(dir_name, name.split(".json")[0] + ".json")
        return filename


Instruction:
Here you have a function in Python 3, explain what it does
def do_load(self, design, init=False):
        # type: (str, bool) -> None
        """Load a design name, running the child LoadHooks.

        Args:
            design: Name of the design json file, without extension
            init: Passed to the LoadHook to tell the children if this is being
                run at Init or not
        """
        if design:
            filename = self._validated_config_filename(design)
            with open(filename, "r") as f:
                text = f.read()
            structure = json_decode(text)
        else:
            structure = {}
        # Attributes and Children used to be merged, support this
        attributes = structure.get("attributes", structure)
        children = structure.get("children", structure)
        # Set the layout table
        name, mri, x, y, visible = [], [], [], [], []
        for part_name, d in attributes.get("layout", {}).items():
            name.append(part_name)
            mri.append("")
            x.append(d["x"])
            y.append(d["y"])
            visible.append(d["visible"])
        self.set_layout(LayoutTable(name, mri, x, y, visible))
        # Set the exports table
        source, export = [], []
        for source_name, export_name in attributes.get("exports", {}).items():
            source.append(source_name)
            export.append(export_name)
        self.exports.set_value(ExportTable(source, export))
        # Set other attributes
        our_values = {k: v for k, v in attributes.items()
                      if k in self.our_config_attributes}
        block = self.block_view()
        block.put_attribute_values(our_values)
        # Run the load hook to get parts to load their own structure
        self.run_hooks(
            LoadHook(p, c, children.get(p.name, {}), init)
            for p, c in self.create_part_contexts(only_visible=False).items())
        self._mark_clean(design, init)

Response: Load a design name and run the child LoadHooks.


Instruction:
Implement a Python 3 function for
registering a function to be added to the block

Response:
def add_method_model(self,
                         func,  # type: Callable
                         name=None,  # type: Optional[str]
                         description=None,  # type: Optional[str]
                         owner=None,  # type: object
                         ):
        # type: (...) -> MethodModel
        """Register a function to be added to the block"""
        if name is None:
            name = func.__name__
        method = MethodModel.from_callable(func, description)
        self._add_field(owner, name, method, func)
        return method


Instruction:
Implement a function in Python 3 to
register a function to be added to the Block as a MethodModel.

Response:
def add_method_model(self,
                         func,  # type: Callable
                         name=None,  # type: Optional[str]
                         description=None,  # type: Optional[str]
                         ):
        # type: (...) -> MethodModel
        """Register a function to be added to the Block as a MethodModel"""
        return self._field_registry.add_method_model(
            func, name, description, self._part)


Instruction:
Implement a Python 3 function for
registering a pre - existing AttributeModel to be added to the Block.

Response:
def add_attribute_model(self,
                            name,  # type: str
                            attr,  # type: AttributeModel
                            writeable_func=None,  # type: Optional[Callable]
                            ):
        # type: (...) -> AttributeModel
        """Register a pre-existing AttributeModel to be added to the Block"""
        return self._field_registry.add_attribute_model(
            name, attr, writeable_func, self._part)


Instruction:
Can you write a function in Python 3 where it
adds variables to the internal list of variables.

Response:
def add_variables(
        self, variables: Union[List["Variable"], "Variable"], index: int = None
    ) -> None:
        """
        Parameters
        ----------
        variable : list of variables or single variable
            Add variables as problem variables

        index : int
            Location to add variables, if None add to the end

        """
        if isinstance(variables, Variable):
            addvars = copy.deepcopy([variables])
        else:
            addvars = copy.deepcopy(variables)

        if index is None:
            self.variables.extend(addvars)
        else:
            self.variables[index:index] = addvars


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _centroids(n_clusters: int, points: List[List[float]]) -> List[List[float]]:

    k_means = KMeans(n_clusters=n_clusters)
    k_means.fit(points)

    closest, _ = pairwise_distances_argmin_min(k_means.cluster_centers_, points)

    return list(map(list, np.array(points)[closest.tolist()]))

Response: Return n_clusters centroids of points


Instruction:
Explain what the following Python 3 code does
def random_weights(nobj: int, nweight: int) -> List[List[float]]:

    # Initial wector space as per
    # Miettinen, K. Nonlinear Multiobjective Optimization
    # Kluwer Academic Publishers, 1999
    wspace = 50 * nobj
    while wspace < nweight:
        wspace *= 2

    weights = np.random.rand(wspace, nobj)
    return _centroids(nobj, weights)

Response: Generate random weights for nof objectives


Instruction:
Can you tell what is the following Python 3 function doing
def new_points(
    factory: IterationPointFactory, solution, weights: List[List[float]] = None
) -> List[Tuple[np.ndarray, List[float]]]:
    """Generate approximate set of points

    Generate set of Pareto optimal solutions projecting from the Pareto optimal solution
    using weights to determine the direction.

    Parameters
    ----------

    factory:
        IterationPointFactory with suitable optimization problem

    solution:
        Current solution from which new solutions are projected

    weights:
        Direction of the projection, if not given generate with
        :func:random_weights

    """
    from desdeo.preference.direct import DirectSpecification

    points = []
    nof = factory.optimization_method.optimization_problem.problem.nof_objectives()
    if not weights:
        weights = random_weights(nof, 50 * nof)
    for pref in map(
        lambda w: DirectSpecification(factory.optimization_method, np.array(w)), weights
    ):
        points.append(factory.result(pref, solution))
    return points

Response: Generate approximate set of points for a new projecting from the Pareto optimal solution.


Instruction:
How would you explain what the following Python 3 function does
def as_minimized(values: List[float], maximized: List[bool]) -> List[float]:
    return [v * -1. if m else v for v, m in zip(values, maximized)]

Response: Return vector values as minimized


Instruction:
Create a Python 3 function to
wrap for prompt and prompt_wrapper.

Response:
def _prompt_wrapper(message, default=None, validator=None):
    """ Handle references piped from file
    """

    class MockDocument:

        def __init__(self, text):
            self.text = text

    if HAS_INPUT:
        ret = prompt(message, default=default, validator=validator)
    else:
        ret = sys.stdin.readline().strip()
        print(message, ret)
        if validator:
            validator.validate(MockDocument(ret))
    if "q" in ret:
        if not HAS_OUTPUT:
            print("User exit")
        sys.exit("User exit")
    return ret


Instruction:
Can you tell what is the following Python 3 function doing
def init_nautilus(method):

    print("Preference elicitation options:")
    print("\t1 - Percentages")
    print("\t2 - Relative ranks")
    print("\t3 - Direct")

    PREFCLASSES = [PercentageSpecifictation, RelativeRanking, DirectSpecification]

    pref_sel = int(
        _prompt_wrapper(
            "Reference elicitation ",
            default=u"%s" % (1),
            validator=NumberValidator([1, 3]),
        )
    )

    preference_class = PREFCLASSES[pref_sel - 1]

    print("Nadir: %s" % method.problem.nadir)
    print("Ideal: %s" % method.problem.ideal)

    if method.current_iter - method.user_iters:
        finished_iter = method.user_iters - method.current_iter
    else:
        finished_iter = 0
    new_iters = int(
        _prompt_wrapper(
            u"Ni: ", default=u"%s" % (method.current_iter), validator=NumberValidator()
        )
    )
    method.current_iter = new_iters
    method.user_iters = finished_iter + new_iters

    return preference_class

Response:
Initialize nautilus method

    Parameters
    ----------

    method
        Interactive method used for the process

    Returns
    -------

    PreferenceInformation subclass to be initialized


Instruction:
Write a Python 3 function that can
iterate NAUTILUS method interactively or using given preferences if given.

Response:
def iter_nautilus(method):
    """ Iterate NAUTILUS method either interactively, or using given preferences if given

    Parameters
    ----------

    method : instance of NAUTILUS subclass
       Fully initialized NAUTILUS method instance

    """
    solution = None
    while method.current_iter:
        preference_class = init_nautilus(method)
        pref = preference_class(method, None)
        default = ",".join(map(str, pref.default_input()))
        while method.current_iter:
            method.print_current_iteration()

            pref_input = _prompt_wrapper(
                u"Preferences: ",
                default=default,
                validator=VectorValidator(method, pref),
            )

            cmd = _check_cmd(pref_input)
            if cmd:
                solution = method.zh
                break
            pref = preference_class(
                method, np.fromstring(pref_input, dtype=np.float, sep=",")
            )
            default = ",".join(map(str, pref.pref_input))
            solution, _ = method.next_iteration(pref)
        if cmd and list(cmd)[0] == "c":
            break
    return solution


Instruction:
Write a Python 3 function for
checking that value is in values

Response:
def isin(value, values):
    """ Check that value is in values """
    for i, v in enumerate(value):
        if v not in np.array(values)[:, i]:
            return False
    return True


Instruction:
Write a Python 3 script for
generating n solutions which attempt to trade - off objs1 and objs2.

Response:
def between(self, objs1: List[float], objs2: List[float], n=1):
        """
        Generate `n` solutions which attempt to trade-off `objs1` and `objs2`.

        Parameters
        ----------
        objs1
            First boundary point for desired objective function values

        objs2
            Second boundary point for desired objective function values

        n
            Number of solutions to generate
        """
        from desdeo.preference.base import ReferencePoint

        objs1_arr = np.array(objs1)
        objs2_arr = np.array(objs2)
        segments = n + 1
        diff = objs2_arr - objs1_arr
        solutions = []
        for x in range(1, segments):
            btwn_obj = objs1_arr + float(x) / segments * diff
            solutions.append(
                self._get_ach().result(ReferencePoint(self, btwn_obj), None)
            )
        return ResultSet(solutions)


Instruction:
How would you implement a function in Python 3 that
finds the first focusable window within the node tree

Response:
def find_focusable(node):
    """
    Search for the first focusable window within the node tree
    """

    if not node.children:
        return node

    if node.focus:
        return find_focusable(node.children_dict[node.focus[0]])


Instruction:
Can you generate a brief explanation for the following Python 3 code
def find_parent_split(node, orientation):

    if (node and node.orientation == orientation
        and len(node.children) > 1):
        return node

    if not node or node.type == "workspace":
        return None

    return find_parent_split(node.parent, orientation)

Response: Find the first parent split relative to the given node according to the desired orientation.


Instruction:
How would you code a function in Python 3 to
cycle through windows of the current workspace.

Response:
def cycle_windows(tree, direction):
    """
    Cycle through windows of the current workspace
    """
    wanted = {
        "orientation": ("vertical" if direction in ("up", "down")
                        else "horizontal"),
        "direction": (1 if direction in ("down", "right")
                      else -1),
    }
    split = find_parent_split(tree.focused.parent, wanted["orientation"])

    if split:
        # Get the next child given the direction
        child_ids = [child.id for child in split.children]
        focus_idx = child_ids.index(split.focused_child.id)
        next_idx = (focus_idx + wanted['direction']) % len(child_ids)
        next_node = split.children[next_idx]
        return find_focusable(next_node)
    return None


Instruction:
Make a summary of the following Python 3 code
def cycle_outputs(tree, direction):
    direction = 1 if direction == "next" else -1
    outputs = [output for output in tree.root.children
               if output.name != "__i3"]
    focus_idx = outputs.index(tree.root.focused_child)
    next_idx = (focus_idx + direction) % len(outputs)
    next_output = outputs[next_idx]
    return find_focusable(next_output)

Response: Cycle through the outputs of a tree.


Instruction:
Can you create a Python 3 function that
returns functions with the class

Response:
def with_class(self, cls):
        """ Return functions with the class
        """
        rcls = []
        for key, value in self._classification.items():
            if value[0] == cls:
                rcls.append(key)
        return rcls


Instruction:
Can you generate the documentation for the following Python 3 function
def _as_reference_point(self) -> np.ndarray:
        ref_val = []
        for fn, f in self._classification.items():
            if f[0] == "<":
                ref_val.append(self._method.problem.ideal[fn])
            elif f[0] == "<>":
                ref_val.append(self._method.problem.nadir[fn])
            else:
                ref_val.append(f[1])

        return np.array(ref_val)

Response: Return classification information as reference point


Instruction:
Write a Python 3 function for
solving River Pollution problem with NAUTILUS V1 and E - NAUTILUS Methods

Response:
def main(logfile=False):
    """ Solve River Pollution problem with NAUTILUS V1 and E-NAUTILUS Methods
    """
    # Duplicate output to log file

    class NAUTILUSOptionValidator(Validator):

        def validate(self, document):
            if document.text not in "ao":
                raise ValidationError(
                    message="Please select a for apriori or o for optimization option",
                    cursor_position=0,
                )

    if logfile:
        Tee(logfile)
    first = True
    current_iter = 0
    while first or current_iter:
        # SciPy breaks box constraints

        nautilus_v1 = NAUTILUSv1(RiverPollution(), SciPyDE)
        if not first:
            nautilus_v1.current_iter = current_iter
        first = False
        nadir = nautilus_v1.problem.nadir
        ideal = nautilus_v1.problem.ideal

        solution = tui.iter_nautilus(nautilus_v1)

        current_iter = nautilus_v1.current_iter

        # TODO: Move to tui module
        method_e = None
        if current_iter > 0:
            option = _prompt_wrapper(
                "select a for apriori or o for optimization option: ",
                default="o",
                validator=NAUTILUSOptionValidator(),
            )
            if option.lower() == "a":
                wi = _prompt_wrapper(
                    "Number of PO solutions (10 or 20): ",
                    default="20",
                    validator=tui.NumberValidator(),
                )
                weights = WEIGHTS[wi]

                factory = IterationPointFactory(
                    SciPyDE(NautilusAchievementProblem(RiverPollution()))
                )
                points = misc.new_points(factory, solution, weights=weights)

                method_e = ENAUTILUS(PreGeneratedProblem(points=points), PointSearch)
                method_e.zh_prev = solution

            else:
                method_e = ENAUTILUS(RiverPollution(), SciPyDE)

            # method_e.zh = solution
            method_e.current_iter = nautilus_v1.current_iter
            method_e.user_iters = nautilus_v1.user_iters

            print(
                "E-NAUTILUS\nselected iteration point: %s:"
                % ",".join(map(str, solution))
            )

            while method_e and method_e.current_iter > 0:
                if solution is None:
                    solution = method_e.problem.nadir

                method_e.problem.nadir = nadir
                method_e.problem.ideal = ideal
                cmd = tui.iter_enautilus(
                    method_e, initial_iterpoint=solution, initial_bound=method_e.fh_lo
                )
                if cmd:
                    print(method_e.current_iter)
                    current_iter = method_e.current_iter
                    break
    if tui.HAS_INPUT:
        input("Press ENTER to exit")


Instruction:
Can you create a Python 3 function that
adds a link to the sitemap

Response:
def add_html_link(app, pagename, templatename, context, doctree):
    """As each page is built, collect page names for the sitemap"""
    app.sitemap_links.append(pagename + ".html")


Instruction:
Create a Python 3 function for
generating the sitemap. xml from the collected HTML page links and the language of the collected HTML page links.

Response:
def create_sitemap(app, exception):
    """Generates the sitemap.xml from the collected HTML page links"""
    site_url = app.builder.config.site_url or app.builder.config.html_baseurl
    if not site_url:
        print("sphinx-sitemap error: neither html_baseurl nor site_url "
              "are set in conf.py. Sitemap not built.")
        return
    if (not app.sitemap_links):
        print("sphinx-sitemap warning: No pages generated for sitemap.xml")
        return

    ET.register_namespace('xhtml', "http://www.w3.org/1999/xhtml")

    root = ET.Element("urlset")
    root.set("xmlns", "http://www.sitemaps.org/schemas/sitemap/0.9")

    get_locales(app, exception)

    if app.builder.config.version:
        version = app.builder.config.version + '/'
    else:
        version = app.builder.config.version

    for link in app.sitemap_links:
        url = ET.SubElement(root, "url")
        if app.builder.config.language is not None:
            ET.SubElement(url, "loc").text = site_url + \
                  app.builder.config.language + '/' + version + link
            if len(app.locales) > 0:
                for lang in app.locales:
                    linktag = ET.SubElement(
                        url,
                        "{http://www.w3.org/1999/xhtml}link"
                    )
                    linktag.set("rel", "alternate")
                    linktag.set("hreflang", lang)
                    linktag.set("href", site_url + lang + '/' + version + link)
        elif app.builder.config.version:
            ET.SubElement(url, "loc").text = site_url + version + link
        else:
            ET.SubElement(url, "loc").text = site_url + link

    filename = app.outdir + "/sitemap.xml"
    ET.ElementTree(root).write(filename,
                               xml_declaration=True,
                               encoding='utf-8',
                               method="xml")
    print("sitemap.xml was generated for URL %s in %s" % (site_url, filename))


Instruction:
Here you have a function in Python 3, explain what it does
def estimate_payoff_table(
    opt_meth_cls: Type[OptimizationMethod], mo_prob: MOProblem
) -> Tuple[List[float], List[float]]:
    """
    Estimates the ideal and nadir by using a payoff table. This should give a
    good estimate for the ideal, but can be very inaccurate for the nadir. For an explanation of why, see [DEB2010]_.

    References
    ----------

    .. [DEB2010] Deb, K., Miettinen, K., & Chaudhuri, S. (2010).
        Toward an estimation of nadir objective vector using a hybrid of evolutionary and local search approaches.
        IEEE Transactions on Evolutionary Computation, 14(6), 821-841.
    """
    ideal = [float("inf")] * mo_prob.nobj
    nadir = [float("-inf")] * mo_prob.nobj
    for i in range(mo_prob.nobj):
        meth = opt_meth_cls(SelectedOptimizationProblem(mo_prob, i))
        _decis, obj = meth.search()
        for j, obj_val in enumerate(obj):
            if obj_val < ideal[j]:
                ideal[j] = obj_val
            if obj_val > nadir[j]:
                nadir[j] = obj_val
    return ideal, nadir

Response: Estimate the ideal and nadir by using a payoff table.


Instruction:
Create a Python 3 function for
padding an ideal and nadir estimate.

Response:
def pad(idnad: Tuple[List[float], List[float]], pad_nadir=0.05, pad_ideal=0.0):
    """
    Pad an ideal/nadir estimate. This is mainly useful for padding the nadir
    estimated by a payoff table for safety purposes.
    """
    ideal, nadir = idnad
    ideal_arr = np.array(ideal)
    nadir_arr = np.array(nadir)
    idnad_range = nadir_arr - ideal_arr
    nadir_arr += pad_nadir * idnad_range
    ideal_arr -= pad_ideal * idnad_range
    return list(ideal_arr), list(nadir_arr)


Instruction:
Explain what the following Python 3 code does
def round_off(idnad: Tuple[List[float], List[float]], dp: int = 2):
    ideal, nadir = idnad
    mult = np.power(10, dp)
    ideal_arr = np.floor(np.array(ideal) * mult) / mult
    nadir_arr = np.ceil(np.array(nadir) * mult) / mult
    return list(ideal_arr), list(nadir_arr)

Response: Round off an ideal and nadir estimate e. g. so that it s looks nicer when it s plotted.


Instruction:
Can you generate the documentation for the following Python 3 function
def default_estimate(
    opt_meth: Type[OptimizationMethod], mo_prob: MOProblem, dp: int = 2
) -> Tuple[List[float], List[float]]:
    """
    The recommended nadir/ideal estimator - use a payoff table and then round
    off the result.
    """
    return round_off(estimate_payoff_table(opt_meth, mo_prob), dp)

Response: Default estimate for a nadir orideal node.


Instruction:
Write a Python 3 function that can
search for the optimal solution of the logarithmic logarithm.

Response:
def search(self, max=False, **params) -> Tuple[np.ndarray, List[float]]:
        """
        Search for the optimal solution

        This sets up the search for the optimization and calls the _search method

        Parameters
        ----------
        max : bool (default False)
            If true find mximum of the objective function instead of minimum

        **params : dict [optional]
            Parameters for single objective optimization method
        """

        self._max = max
        if max:
            self._coeff = -1.0
        else:
            self._coeff = 1.0

        return self._search(**params)


Instruction:
Here you have a function in Python 3, explain what it does
def next_iteration(self, preference=None):
        if preference:
            self.preference = preference
            print(("Given preference: %s" % self.preference.pref_input))
        self._update_fh()

        # tmpzh = list(self.zh)
        self._update_zh(self.zh, self.fh)
        # self.zh = list(np.array(self.zh) / 2. + np.array(self.zh_prev) / 2.)
        # self.zh_prev = tmpzh
        if self.current_iter != 1:
            self.fh_lo = list(self.lower_bounds_factory.result(self.zh_prev))

        self.current_iter -= 1

        return self.fh_lo, self.zh

Response: Return next iteration bounds


Instruction:
How would you explain what the following Python 3 function does
def next_iteration(self, ref_point, bounds=None):
        if bounds:
            self.problem.points = reachable_points(
                self.problem.points, self.problem.ideal, bounds
            )
        if not utils.isin(self.fh, self.problem.points) or ref_point != self.ref_point:
            self.ref_point = list(ref_point)
            self._update_fh()

        self._update_zh(self.zh, self.fh)

        self.fh_lo = list(self.lower_bounds_factory.result(self.zh))
        self.fh_up = list(self.upper_bounds_factory.result(self.zh))

        logging.debug(f"Updated upper boundary: {self.fh_up}")
        logging.debug(f"Uppadet lower boundary: {self.fh_lo}")

        if not np.all(np.array(self.fh_up) > np.array(self.fh_lo)):
            warn(self.NegativeIntervalWarning())

        assert utils.isin(self.fh_up, self.problem.points)
        assert utils.isin(self.fh_lo, self.problem.points)

        dist = self.distance(self.zh, self.fh)

        # Reachable points
        self.update_points()

        lP = len(self.problem.points)
        self.current_iter -= 1

        return dist, self.fh, self.zh, self.fh_lo, self.fh_up, lP

Response: Calculate the next iteration point to be shown to the DM.


Instruction:
Make a summary of the following Python 3 code
def output_default(paragraphs, fp=sys.stdout, no_boilerplate=True):
    for paragraph in paragraphs:
        if paragraph.class_type == 'good':
            if paragraph.heading:
                tag = 'h'
            else:
                tag = 'p'
        elif no_boilerplate:
            continue
        else:
            tag = 'b'

        print('<%s> %s' % (tag, cgi.escape(paragraph.text)), file=fp)

Response: Prints the paragraphs as a default list of paragraphs.


Instruction:
Make a summary of the following Python 3 code
def output_detailed(paragraphs, fp=sys.stdout):
    for paragraph in paragraphs:
        output = '<p class="%s" cfclass="%s" heading="%i" xpath="%s"> %s' % (
            paragraph.class_type,
            paragraph.cf_class,
            int(paragraph.heading),
            paragraph.xpath,
            cgi.escape(paragraph.text)
        )
        print(output, file=fp)

Response: Output detailed information about the given list of paragraphs.


Instruction:
Given the following Python 3 function, write the documentation
def output_krdwrd(paragraphs, fp=sys.stdout):
    for paragraph in paragraphs:
        if paragraph.class_type in ('good', 'neargood'):
            if paragraph.heading:
                cls = 2
            else:
                cls = 3
        else:
            cls = 1

        for text_node in paragraph.text_nodes:
            print('%i\t%s' % (cls, text_node.strip()), file=fp)

Response: Outputs the given list of paragraphs in a KrdWrd compatible format.


Instruction:
Here you have a function in Python 3, explain what it does
def html_to_dom(html, default_encoding=DEFAULT_ENCODING, encoding=None, errors=DEFAULT_ENC_ERRORS):
    if isinstance(html, unicode):
        decoded_html = html
        # encode HTML for case it's XML with encoding declaration
        forced_encoding = encoding if encoding else default_encoding
        html = html.encode(forced_encoding, errors)
    else:
        decoded_html = decode_html(html, default_encoding, encoding, errors)

    try:
        dom = lxml.html.fromstring(decoded_html, parser=lxml.html.HTMLParser())
    except ValueError:
        # Unicode strings with encoding declaration are not supported.
        # for XHTML files with encoding declaration, use the declared encoding
        dom = lxml.html.fromstring(html, parser=lxml.html.HTMLParser())

    return dom

Response: Converts HTML to DOM.


Instruction:
Can you generate the documentation for the following Python 3 function
def decode_html(html, default_encoding=DEFAULT_ENCODING, encoding=None, errors=DEFAULT_ENC_ERRORS):
    if isinstance(html, unicode):
        return html

    if encoding:
        return html.decode(encoding, errors)

    match = CHARSET_META_TAG_PATTERN.search(html)
    if match:
        declared_encoding = match.group(1).decode("ASCII")
        # proceed unknown encoding as if it wasn't found at all
        with ignored(LookupError):
            return html.decode(declared_encoding, errors)

    # unknown encoding
    try:
        # try UTF-8 first
        return html.decode("utf8")
    except UnicodeDecodeError:
        # try lucky with default encoding
        try:
            return html.decode(default_encoding, errors)
        except UnicodeDecodeError as e:
            raise JustextError("Unable to decode the HTML to Unicode: " + unicode(e))

Response: Converts a HTML page into Unicode.


Instruction:
Write a Python 3 script to
remove unwanted parts of DOM.

Response:
def preprocessor(dom):
    "Removes unwanted parts of DOM."
    options = {
        "processing_instructions": False,
        "remove_unknown_tags": False,
        "safe_attrs_only": False,
        "page_structure": False,
        "annoying_tags": False,
        "frames": False,
        "meta": False,
        "links": False,
        "javascript": False,
        "scripts": True,
        "comments": True,
        "style": True,
        "embedded": True,
        "forms": True,
        "kill_tags": ("head",),
    }
    cleaner = Cleaner(**options)

    return cleaner.clean_html(dom)


Instruction:
How would you code a function in Python 3 to
get the next neighbour of the given paragraphs.

Response:
def get_next_neighbour(i, paragraphs, ignore_neargood):
    """
    Return the class of the paragraph at the bottom end of the short/neargood
    paragraphs block. If ignore_neargood is True, than only 'bad' or 'good'
    can be returned, otherwise 'neargood' can be returned, too.
    """
    return _get_neighbour(i, paragraphs, ignore_neargood, 1, len(paragraphs))


Instruction:
Can you tell what is the following Python 3 function doing
def revise_paragraph_classification(paragraphs, max_heading_distance=MAX_HEADING_DISTANCE_DEFAULT):
    # copy classes
    for paragraph in paragraphs:
        paragraph.class_type = paragraph.cf_class

    # good headings
    for i, paragraph in enumerate(paragraphs):
        if not (paragraph.heading and paragraph.class_type == 'short'):
            continue
        j = i + 1
        distance = 0
        while j < len(paragraphs) and distance <= max_heading_distance:
            if paragraphs[j].class_type == 'good':
                paragraph.class_type = 'neargood'
                break
            distance += len(paragraphs[j].text)
            j += 1

    # classify short
    new_classes = {}
    for i, paragraph in enumerate(paragraphs):
        if paragraph.class_type != 'short':
            continue
        prev_neighbour = get_prev_neighbour(i, paragraphs, ignore_neargood=True)
        next_neighbour = get_next_neighbour(i, paragraphs, ignore_neargood=True)
        neighbours = set((prev_neighbour, next_neighbour))
        if neighbours == set(['good']):
            new_classes[i] = 'good'
        elif neighbours == set(['bad']):
            new_classes[i] = 'bad'
        # it must be set(['good', 'bad'])
        elif (prev_neighbour == 'bad' and get_prev_neighbour(i, paragraphs, ignore_neargood=False) == 'neargood') or \
             (next_neighbour == 'bad' and get_next_neighbour(i, paragraphs, ignore_neargood=False) == 'neargood'):
            new_classes[i] = 'good'
        else:
            new_classes[i] = 'bad'

    for i, c in new_classes.items():
        paragraphs[i].class_type = c

    # revise neargood
    for i, paragraph in enumerate(paragraphs):
        if paragraph.class_type != 'neargood':
            continue
        prev_neighbour = get_prev_neighbour(i, paragraphs, ignore_neargood=True)
        next_neighbour = get_next_neighbour(i, paragraphs, ignore_neargood=True)
        if (prev_neighbour, next_neighbour) == ('bad', 'bad'):
            paragraph.class_type = 'bad'
        else:
            paragraph.class_type = 'good'

    # more good headings
    for i, paragraph in enumerate(paragraphs):
        if not (paragraph.heading and paragraph.class_type == 'bad' and paragraph.cf_class != 'bad'):
            continue
        j = i + 1
        distance = 0
        while j < len(paragraphs) and distance <= max_heading_distance:
            if paragraphs[j].class_type == 'good':
                paragraph.class_type = 'good'
                break
            distance += len(paragraphs[j].text)
            j += 1

Response: This function revise the context - sensitive paragraph classification.


Instruction:
Write a Python 3 script for
converting an HTML page into a list of classified paragraphs.

Response:
def justext(html_text, stoplist, length_low=LENGTH_LOW_DEFAULT,
        length_high=LENGTH_HIGH_DEFAULT, stopwords_low=STOPWORDS_LOW_DEFAULT,
        stopwords_high=STOPWORDS_HIGH_DEFAULT, max_link_density=MAX_LINK_DENSITY_DEFAULT,
        max_heading_distance=MAX_HEADING_DISTANCE_DEFAULT, no_headings=NO_HEADINGS_DEFAULT,
        encoding=None, default_encoding=DEFAULT_ENCODING,
        enc_errors=DEFAULT_ENC_ERRORS, preprocessor=preprocessor):
    """
    Converts an HTML page into a list of classified paragraphs. Each paragraph
    is represented as instance of class ˙˙justext.paragraph.Paragraph˙˙.
    """
    dom = html_to_dom(html_text, default_encoding, encoding, enc_errors)
    dom = preprocessor(dom)

    paragraphs = ParagraphMaker.make_paragraphs(dom)

    classify_paragraphs(paragraphs, stoplist, length_low, length_high,
        stopwords_low, stopwords_high, max_link_density, no_headings)
    revise_paragraph_classification(paragraphs, max_heading_distance)

    return paragraphs


Instruction:
Can you write a function in Python 3 where it
converts DOM into paragraphs.

Response:
def make_paragraphs(cls, root):
        """Converts DOM into paragraphs."""
        handler = cls()
        lxml.sax.saxify(root, handler)
        return handler.paragraphs


Instruction:
Create a Python 3 function for
returning a collection of built - in stop - lists.

Response:
def get_stoplists():
    """Returns a collection of built-in stop-lists."""
    path_to_stoplists = os.path.dirname(sys.modules["justext"].__file__)
    path_to_stoplists = os.path.join(path_to_stoplists, "stoplists")

    stoplist_names = []
    for filename in os.listdir(path_to_stoplists):
        name, extension = os.path.splitext(filename)
        if extension == ".txt":
            stoplist_names.append(name)

    return frozenset(stoplist_names)


Instruction:
Explain what the following Python 3 code does
def get_stoplist(language):
    file_path = os.path.join("stoplists", "%s.txt" % language)
    try:
        stopwords = pkgutil.get_data("justext", file_path)
    except IOError:
        raise ValueError(
            "Stoplist for language '%s' is missing. "
            "Please use function 'get_stoplists' for complete list of stoplists "
            "and feel free to contribute by your own stoplist." % language
        )

    return frozenset(w.decode("utf8").lower() for w in stopwords.splitlines())

Response: Returns an empty stop - list for the language as a set of words.


Instruction:
Explain what the following Python 3 code does
def get_boto_client(
        client,
        region=None,
        aws_access_key_id=None,
        aws_secret_access_key=None,
        aws_session_token=None,
        endpoint_url=None
        ):
    """Get a boto3 client connection."""
    cache_key = '{0}:{1}:{2}:{3}'.format(
        client,
        region,
        aws_access_key_id,
        endpoint_url or ''
    )
    if not aws_session_token:
        if cache_key in CLIENT_CACHE:
            return CLIENT_CACHE[cache_key]
    session = get_boto_session(
        region,
        aws_access_key_id,
        aws_secret_access_key,
        aws_session_token
    )
    if not session:
        logging.error("Failed to get {0} client.".format(client))
        return None

    CLIENT_CACHE[cache_key] = session.client(
        client,
        endpoint_url=endpoint_url
    )
    return CLIENT_CACHE[cache_key]

Response: Get a boto3 client connection.


Instruction:
Implement a Python 3 function for
getting a boto resource connection.

Response:
def get_boto_resource(
        resource,
        region=None,
        aws_access_key_id=None,
        aws_secret_access_key=None,
        aws_session_token=None,
        endpoint_url=None
        ):
    """Get a boto resource connection."""
    cache_key = '{0}:{1}:{2}:{3}'.format(
        resource,
        region,
        aws_access_key_id,
        endpoint_url or ''
    )
    if not aws_session_token:
        if cache_key in RESOURCE_CACHE:
            return RESOURCE_CACHE[cache_key]
    session = get_boto_session(
        region,
        aws_access_key_id,
        aws_secret_access_key,
        aws_session_token
    )
    if not session:
        logging.error("Failed to get {0} resource.".format(resource))
        return None

    RESOURCE_CACHE[cache_key] = session.resource(
        resource,
        endpoint_url=endpoint_url
    )
    return RESOURCE_CACHE[cache_key]


Instruction:
Can you generate a brief explanation for the following Python 3 code
def get_boto_session(
        region,
        aws_access_key_id=None,
        aws_secret_access_key=None,
        aws_session_token=None
        ):
    """Get a boto3 session."""
    return boto3.session.Session(
        region_name=region,
        aws_secret_access_key=aws_secret_access_key,
        aws_access_key_id=aws_access_key_id,
        aws_session_token=aws_session_token
    )

Response: Get a boto3 session.


Instruction:
Write a Python 3 script to
ensure an input is a string decoding if it is bytes.

Response:
def ensure_text(str_or_bytes, encoding='utf-8'):
    """Ensures an input is a string, decoding if it is bytes.
    """
    if not isinstance(str_or_bytes, six.text_type):
        return str_or_bytes.decode(encoding)
    return str_or_bytes


Instruction:
How would you explain what the following Python 3 function does
def ensure_bytes(str_or_bytes, encoding='utf-8', errors='strict'):
    if isinstance(str_or_bytes, six.text_type):
        return str_or_bytes.encode(encoding, errors)
    return str_or_bytes

Response: Ensures an input is bytes encoding if it is a string.