Instruction:
Implement a function in Python 3 to
filter DataFrame using filter_ key - value conditions applying logic_operation only find rows strictly fitting the filter_ criterion

Response:
def __filter(filterable, filter_, logic_operation='and'):
        """ filtering DataFrame using filter_ key-value conditions applying logic_operation
        only find rows strictly fitting the filter_ criterion"""
        condition = []
        if not filter_:
            return filterable
        elif filter_.get('type') == '__ANY__':
            return filterable
        else:
            for key, value in filter_.items():
                condition.append('{key} == "{value}"'.format(key=key, value=value))
        try:
            res = filterable.query(" {operation} ".format(operation=logic_operation).join(condition))
        except pd.core.computation.ops.UndefinedVariableError:
            return pd.DataFrame()
        else:
            return res


Instruction:
Write a Python 3 script to
reverse filtering DataFrame using filter_ key - value conditions applying logic_operation

Response:
def __reversed_filter(filterable, filter_, logic_operation='and'):
        """ reverse filtering DataFrame using filter_ key-value conditions applying logic_operation
        find rows where existing filterable columns (and its values) fitting the filter_ criterion"""
        condition = []
        try:
            subscribers_for_any = filterable.query('type == "__ANY__"')
        except pd.core.computation.ops.UndefinedVariableError:
            subscribers_for_any = pd.DataFrame()
        if not filter_:
            return filterable
        else:
            for existing_col in filterable:
                for meta_tag, meta_value in filter_.items():
                    if meta_tag == existing_col:
                        condition.append('{key} == "{value}"'.format(key=meta_tag, value=meta_value))
            try:
                res = filterable.query(" {operation} ".format(operation=logic_operation).join(condition))
            except pd.core.computation.ops.UndefinedVariableError:
                return pd.DataFrame().append(subscribers_for_any)
            else:
                return res.append(subscribers_for_any)


Instruction:
Can you generate the documentation for the following Python 3 function
def get(self, key):
        value = self._redis.get(self._prefix + key)

        if value is not None:
            return self.unserialize(value)

Response: Retrieves an item from the cache by key.


Instruction:
Given the following Python 3 function, write the documentation
def put(self, key, value, minutes):
        value = self.serialize(value)

        minutes = max(1, minutes)

        self._redis.setex(self._prefix + key, minutes * 60, value)

Response: Stores an item in the cache for a given number of minutes.


Instruction:
Write a Python 3 function for
incrementing the value of an item in the cache.

Response:
def increment(self, key, value=1):
        """
        Increment the value of an item in the cache.

        :param key: The cache key
        :type key: str

        :param value: The increment value
        :type value: int

        :rtype: int or bool
        """
        return self._redis.incrby(self._prefix + key, value)


Instruction:
Create a Python 3 function for
storing an item in the cache indefinitely.

Response:
def forever(self, key, value):
        """
        Store an item in the cache indefinitely.

        :param key: The cache key
        :type key: str

        :param value: The value to store
        :type value: mixed
        """
        value = self.serialize(value)

        self._redis.set(self._prefix + key, value)


Instruction:
How would you explain what the following Python 3 function does
def forget(self, key):
        return bool(self._redis.delete(self._prefix + key))

Response: Removes an item from the cache.


Instruction:
Explain what the following Python 3 code does
def get(self, key, default=None):
        val = self._store.get(key)

        if val is None:
            return value(default)

        return val

Response: Retrieves an item from the cache by key.


Instruction:
Explain what the following Python 3 code does
def put(self, key, val, minutes):
        minutes = self._get_minutes(minutes)

        if minutes is not None:
            self._store.put(key, val, minutes)

Response: Stores an item in the cache.


Instruction:
Write a Python 3 function for
storing an item in the cache if it does not exist. Returns True if the item was added False otherwise.

Response:
def add(self, key, val, minutes):
        """
        Store an item in the cache if it does not exist.

        :param key: The cache key
        :type key: str

        :param val: The cache value
        :type val: mixed

        :param minutes: The lifetime in minutes of the cached value
        :type minutes: int|datetime

        :rtype: bool
        """
        if hasattr(self._store, 'add'):
            return self._store.add(key, val, self._get_minutes(minutes))

        if not self.has(key):
            self.put(key, val, minutes)

            return True

        return False


Instruction:
Explain what the following Python 3 code does
def _get_key(self, fn, args, kwargs):
        if args:
            serialized_arguments = (
                self._store.serialize(args[1:])
                + self._store.serialize([(k, kwargs[k]) for k in sorted(kwargs.keys())])
            )
        else:
            serialized_arguments = self._store.serialize([(k, kwargs[k]) for k in sorted(kwargs.keys())])

        if isinstance(fn, types.MethodType):
            key = self._hash('%s.%s.%s'
                             % (fn.__self__.__class__.__name__,
                                args[0].__name__,
                                serialized_arguments))
        elif isinstance(fn, types.FunctionType):
            key = self._hash('%s.%s'
                             % (fn.__name__,
                                serialized_arguments))
        else:
            key = '%s:' % fn + self._hash(serialized_arguments)

        return key

Response: Calculate a cache key given a function args and kwargs.


Instruction:
How would you explain what the following Python 3 function does
def delete(self, path=None, headers=None):
        return self.request("DELETE", path=path, headers=headers)

Response:
HTTP DELETE
        - path: string  additionnal path to the uri
        - headers: dict, optionnal headers that will
            be added to HTTP request.
        - params: Optionnal parameterss added to the request


Instruction:
Write a Python 3 function that can
flush the implicit main function if there is no main function defined.

Response:
def flush_main(self):
        """Flushes the implicit main function if there is no main
        function defined."""

        if self.has_main:
            return

        self.in_main = True

        self.write('task main()')
        self.INDENT()

        if self.debug:
            print 'Flushing main:', self.fv.main
        for node in self.fv.main:
            self.v(node)
            self.NEWLINE()

        self.DEDENT()

        self.in_main = False


Instruction:
How would you explain what the following Python 3 function does
def put(self, key, value, minutes):
        self._memcache.set(self._prefix + key, value, minutes * 60)

Response: Stores an item in the cache for a given number of minutes.


Instruction:
Given the following Python 3 function, write the documentation
def add(self, key, val, minutes):
        return self._memcache.add(self._prefix + key, val, minutes * 60)

Response: Stores an item in the cache if it does not exist. Returns True if the item was added False otherwise.


Instruction:
Create a Python 3 function for
incrementing the value of an item in the cache.

Response:
def increment(self, key, value=1):
        """
        Increment the value of an item in the cache.

        :param key: The cache key
        :type key: str

        :param value: The increment value
        :type value: int

        :rtype: int or bool
        """
        return self._memcache.incr(self._prefix + key, value)


Instruction:
How would you code a function in Python 3 to
collect all immediately available items from a queue

Response:
def get_nowait_from_queue(queue):
    """ Collect all immediately available items from a queue """
    data = []
    for _ in range(queue.qsize()):
        try:
            data.append(queue.get_nowait())
        except q.Empty:
            break
    return data


Instruction:
Here you have a function in Python 3, explain what it does
def forever(self, key, value):
        namespace = self._tags.get_namespace()

        self._push_forever_keys(namespace, key)

        self._store.forever(
            '%s:%s' % (hashlib.sha1(encode(self._tags.get_namespace())).hexdigest(), key),
            value
        )

Response: Store an item in the cache indefinitely.


Instruction:
How would you implement a function in Python 3 that
pushes the full key for each namespace segment.

Response:
def _push_forever_keys(self, namespace, key):
        """
        Store a copy of the full key for each namespace segment.

        :type namespace: str
        :type key: str
        """
        full_key = '%s%s:%s' % (self.get_prefix(),
                                hashlib.sha1(encode(self._tags.get_namespace())).hexdigest(),
                                key)

        for segment in namespace.split('|'):
            self._store.connection().lpush(self._forever_key(segment), full_key)


Instruction:
Can you generate the documentation for the following Python 3 function
def _delete_forever_keys(self):
        for segment in self._tags.get_namespace().split('|'):
            segment = self._forever_key(segment)
            self._delete_forever_values(segment)

            self._store.connection().delete(segment)

Response: Delete all of the items that were stored forever.


Instruction:
Make a summary of the following Python 3 code
def _delete_forever_values(self, forever_key):
        forever = self._store.connection().lrange(forever_key, 0, -1)

        if len(forever) > 0:
            self._store.connection().delete(*forever)

Response: Delete all of the keys that have been stored forever.


Instruction:
Write a Python 3 script for
capturing Python system warnings.

Response:
def capture_pywarnings(handler):
        """Log python system warnings."""
        logger = logging.getLogger('py.warnings')
        # Check for previously installed handlers.
        for h in logger.handlers:
            if isinstance(h, handler.__class__):
                return
        logger.addHandler(handler)
        logger.setLevel(logging.WARNING)


Instruction:
Can you write a function in Python 3 where it
retrieves an item and expiry time from the cache by key.

Response:
def _get_payload(self, key):
        """
        Retrieve an item and expiry time from the cache by key.

        :param key: The cache key
        :type key: str

        :rtype: dict
        """
        payload = self._storage.get(key)

        # If the key does not exist, we return nothing
        if not payload:
            return (None, None)

        expire = payload[0]

        # If the current time is greater than expiration timestamps we will delete
        # the entry
        if round(time.time()) >= expire:
            self.forget(key)

            return (None, None)

        data = payload[1]

        # Next, we'll extract the number of minutes that are remaining for a cache
        # so that we can properly retain the time for things like the increment
        # operation that may be performed on the cache. We'll round this out.
        time_ = math.ceil((expire - round(time.time())) / 60.)

        return (data, time_)


Instruction:
How would you explain what the following Python 3 function does
def put(self, key, value, minutes):
        self._storage[key] = (self._expiration(minutes), value)

Response: Stores an item in the cache for a given number of minutes.


Instruction:
How would you implement a function in Python 3 that
increments the value of an item in the cache.

Response:
def increment(self, key, value=1):
        """
        Increment the value of an item in the cache.

        :param key: The cache key
        :type key: str

        :param value: The increment value
        :type value: int

        :rtype: int or bool
        """
        data, time_ = self._get_payload(key)

        integer = int(data) + value

        self.put(key, integer, int(time_))

        return integer


Instruction:
Make a summary of the following Python 3 code
def forget(self, key):
        if key in self._storage:
            del self._storage[key]

            return True

        return False

Response: Removes an item from the cache.


Instruction:
Write a Python 3 function that can
get detail for a resource from the data endpoint.

Response:
async def get_data(self, resource):
        """Get detail for a resource from the data endpoint."""
        url = '{}{}'.format(
            self.base_url, self.endpoint.format(resource=resource))

        try:
            with async_timeout.timeout(5, loop=self._loop):
                response = await self._session.get(url)

            _LOGGER.info(
                "Response from Netdata: %s", response.status)
            data = await response.json()
            _LOGGER.debug(data)
            self.values = {k: v for k, v in zip(
                data['labels'], data['data'][0])}

        except (asyncio.TimeoutError, aiohttp.ClientError, socket.gaierror):
            _LOGGER.error("Can not load data from Netdata")
            raise exceptions.NetdataConnectionError()


Instruction:
Can you implement a function in Python 3 that
gets alarms for a Netdata instance.

Response:
async def get_alarms(self):
        """Get alarms for a Netdata instance."""
        url = '{}{}'.format(self.base_url, self.endpoint)

        try:
            with async_timeout.timeout(5, loop=self._loop):
                response = await self._session.get(url)

            _LOGGER.debug(
                "Response from Netdata: %s", response.status)
            data = await response.text()
            _LOGGER.debug(data)
            self.alarms = data

        except (asyncio.TimeoutError, aiohttp.ClientError, socket.gaierror):
            _LOGGER.error("Can not load data from Netdata")
            raise exceptions.NetdataConnectionError()


Instruction:
Write a Python 3 script to
add request_id to log record.

Response:
def filter(self, record):
        """If request_id is set in flask.g, add it to log record."""
        if g and hasattr(g, 'request_id'):
            record.request_id = g.request_id
        return True


Instruction:
Make a summary of the following Python 3 code
def init_logging(log_filename, verbose, quiet):
    # TODO: consider making one verbosity parameter instead of two mutually exclusive
    # TODO: default values for parameters
    logger = logging.getLogger('')
    logger.setLevel(logging.DEBUG)

    # add file handler if needed
    if log_filename:
        file_handler = logging.FileHandler(log_filename)
        file_handler.setLevel(logging.DEBUG)
        # TODO: initialize all formatters in the beginning of this function
        file_handler.setFormatter(
            logging.Formatter(
                fmt="%(asctime)s [%(levelname)s] %(name)s %(filename)s:%(lineno)d\t%(message)s"
            ))
        logger.addHandler(file_handler)

    # console stdout and stderr handlers
    console_handler = logging.StreamHandler(sys.stdout)
    stderr_hdl = logging.StreamHandler(sys.stderr)

    # formatters
    fmt_verbose = logging.Formatter(
        fmt="%(asctime)s [%(levelname)s] %(name)s %(filename)s:%(lineno)d\t%(message)s",
        datefmt='%Y-%m-%d,%H:%M:%S.%f'
    )
    fmt_regular = logging.Formatter(
        "%(asctime)s [%(levelname).4s] [%(filename).8s] %(message)s", "%H:%M:%S")

    # set formatters and log levels
    if verbose:
        console_handler.setLevel(logging.DEBUG)
        console_handler.setFormatter(fmt_verbose)
        stderr_hdl.setFormatter(fmt_verbose)
    elif quiet:
        console_handler.setLevel(logging.WARNING)
        console_handler.setFormatter(fmt_regular)
        stderr_hdl.setFormatter(fmt_regular)
    else:
        console_handler.setLevel(logging.INFO)
        console_handler.setFormatter(fmt_regular)
        stderr_hdl.setFormatter(fmt_regular)

    # TODO: do we really need these to be redirected?
    # redirect ERROR, WARNING and CRITICAL to sterr
    f_err = SingleLevelFilter(logging.ERROR, True)
    f_warn = SingleLevelFilter(logging.WARNING, True)
    f_crit = SingleLevelFilter(logging.CRITICAL, True)
    console_handler.addFilter(f_err)
    console_handler.addFilter(f_warn)
    console_handler.addFilter(f_crit)
    logger.addHandler(console_handler)

    f_info = SingleLevelFilter(logging.INFO, True)
    f_debug = SingleLevelFilter(logging.DEBUG, True)
    stderr_hdl.addFilter(f_info)
    stderr_hdl.addFilter(f_debug)
    logger.addHandler(stderr_hdl)

Response: Initialize the logging module with default parameters.


Instruction:
How would you implement a function in Python 3 that
gets a cache store instance by name.

Response:
def store(self, name=None):
        """
        Get a cache store instance by name.

        :param name: The cache store name
        :type name: str

        :rtype: Repository
        """
        if name is None:
            name = self.get_default_driver()

        self._stores[name] = self._get(name)

        return self._stores[name]


Instruction:
Given the following Python 3 function, write the documentation
def _get(self, name):
        return self._stores.get(name, self._resolve(name))

Response: Get the store from the local cache.


Instruction:
Here you have a function in Python 3, explain what it does
def _resolve(self, name):
        config = self._get_config(name)

        if not config:
            raise RuntimeError('Cache store [%s] is not defined.' % name)

        if config['driver'] in self._custom_creators:
            repository = self._call_custom_creator(config)
        else:
            repository = getattr(self, '_create_%s_driver' % config['driver'])(config)

        if 'serializer' in config:
            serializer = self._resolve_serializer(config['serializer'])
        else:
            serializer = self._serializer

        repository.get_store().set_serializer(serializer)

        return repository

Response: Resolve the given store and return a new instance of the appropriate class.


Instruction:
How would you code a function in Python 3 to
call a custom driver creator.

Response:
def _call_custom_creator(self, config):
        """
        Call a custom driver creator.

        :param config: The driver configuration
        :type config: dict

        :rtype: Repository
        """
        creator = self._custom_creators[config['driver']](config)

        if isinstance(creator, Store):
            creator = self.repository(creator)

        if not isinstance(creator, Repository):
            raise RuntimeError('Custom creator should return a Repository instance.')

        return creator


Instruction:
Can you write a function in Python 3 where it
creates an instance of the file cache driver.

Response:
def _create_file_driver(self, config):
        """
        Create an instance of the file cache driver.

        :param config: The driver configuration
        :type config: dict

        :rtype: Repository
        """
        kwargs = {
            'directory': config['path']
        }

        if 'hash_type' in config:
            kwargs['hash_type'] = config['hash_type']

        return self.repository(FileStore(**kwargs))


Instruction:
Can you tell what is the following Python 3 function doing
def get_default_driver(self):
        if 'default' in self._config:
            return self._config['default']

        if len(self._config['stores']) == 1:
            return list(self._config['stores'].keys())[0]

        raise RuntimeError('Missing "default" cache in configuration.')

Response: Get the default cache driver name.


Instruction:
Explain what the following Python 3 code does
def _resolve_serializer(self, serializer):
        if isinstance(serializer, Serializer):
            return serializer

        if serializer in self._serializers:
            return self._serializers[serializer]

        raise RuntimeError('Unsupported serializer')

Response: Resolve the given serializer.


Instruction:
Write a Python 3 function for
generating a single multicommand question that allows the user to select one or more answers.

Response:
def multiple_choice(question, choices, answers):
    """
    Generates a multiple choice question that allows the user to select an
    answer choice and shows whether choice was correct.

    Args:
        question (str): Question text displayed above choices.

        choices (list str): Answer choices that user can select.

        answers (int | iterable int): Either an integer or iterable of
            integers. Each integer in answers corresponds to the index of the
            correct choice in `choices`.

    Returns:
        None

    >>> multiple_choice(question="What is 10 + 2 * 5?",
    ...                 choices=['12', '60', '20'],
    ...                 answers=2) #doctest: +SKIP
    <What is 10 + 2 * 5?>
    <Button> <12>
    <Button> <60>
    <Button> <20>   (Correct)
    >>> multiple_choice(question="Select all prime numbers.",
    ...                 choices=['12', '3', '31'],
    ...                 answers=[1, 2]) #doctest: +SKIP
    <Select all prime numbers.>
    <Button> <12>
    <Button> <3>    (Correct)
    <Button> <31>   (Correct)
    """
    if not isinstance(answers, (int, collections.Iterable)):
        raise TypeError(
            'The `answers` arg is expected to be of type '
            '(int | iterable int) but got {} instead.'.format(type(answers))
        )

    @curry
    def check_answer(index, button):
        is_correct = (
            index == answers if isinstance(answers, int) else index in answers
        )
        button.style.button_color = GREEN if is_correct else RED

    answer_choices = []
    for index, choice in enumerate(choices):
        button = widgets.Button(
            layout=widgets.Layout(width='20px', height='20px', padding='0')
        )
        button.on_click(check_answer(index))

        button_and_question = widgets.HBox(
            [button, widgets.HTML(TEXT_STYLE.format(choice))],
            layout=widgets.Layout(align_items='center')
        )

        answer_choices.append(button_and_question)

    question_html = [widgets.HTML(TEXT_STYLE.format(question))]
    display(widgets.VBox(question_html + answer_choices))


Instruction:
Write a Python 3 function that can
generate a short answer question that allows user to input an answer in the order that they ve entered.

Response:
def short_answer(question, answers, explanation=None):
    """
    Generates a short answer question that allows user to input an answer in
    a textbox and a submit button to check the answer.

    Args:
        question (str):  The question being asked.

        answers (str | list str | func): If a string, only that string will be
            marked correct. If a list of string, any string in the list will be
            marked correct. If a function, any input that causes the function
            to return True will be marked correct.

        explanation (str):  The explanation to the question is displayed when
            the user inputs the correct answer.

    Returns:
        None

    >>> short_answer('What is 1 + 1?', '2',
    ...              explanation='1+1 is 2') #doctest: +SKIP
    <What is 1+1?>
    <Input box, Submit button>
    >>> short_answer('Enter the first name of a member of the Beatles.',
    ...              ['John', 'Paul', 'George', 'Ringo']) #doctest: +SKIP
    <Enter the first name of a member of the Beatles.>
    <Input box, Submit button>
    >>> short_answer('Enter an even number.',
    ...              lambda x: int(x) % 2 == 0) #doctest: +SKIP
    <Enter an even number.>
    <Input box, Submit button>
    """
    # Input textbox
    textbox = widgets.Text(placeholder='Write your answer here')
    # Submit button
    submit_button = widgets.Button(description='Submit')
    # Space right of the submit button to show checkmark/x-mark
    visual_correct = widgets.HTML()
    # Space below input line to display error if function call errored
    error_space = widgets.HTML()
    # Space below input line to display explanation if answer is correct
    explain_space = widgets.HTML()

    # correctness function linked to the submit button
    def check_answer(_):
        response = textbox.value
        if isinstance(answers, collections.Callable):
            try:
                error_space.value = ''
                correct = answers(response)
            except Exception as e:
                correct = False
                error_space.value = 'Error in checking answer: {}'.format(e)
        elif isinstance(answers, str):
            correct = response == answers
        elif isinstance(answers, collections.Iterable):
            correct = response in answers
        else:
            raise TypeError('The `answers` arg is an incorrect type.')

        visual_correct.value = CHECK_ICON if correct else X_ICON
        if correct and explanation:
            explain_space.value = explanation

    submit_button.on_click(check_answer)
    question_tag = widgets.HTML(TEXT_STYLE.format(question))
    user_input_line = widgets.HBox([textbox, submit_button, visual_correct])
    display(
        widgets.VBox([
            question_tag, user_input_line, error_space, explain_space
        ])
    )


Instruction:
Can you create a Python 3 function that
converts a notebook file to an HTML file.

Response:
def publish(spec, nb_name, template='full', save_first=True):
    """
    Converts nb_name to an HTML file. Preserves widget functionality.

    Outputs a link to download HTML file after conversion if called in a
    notebook environment.

    Equivalent to running `nbinteract ${spec} ${nb_name}` on the command line.

    Args:
        spec (str): BinderHub spec for Jupyter image. Must be in the format:
            `${username}/${repo}/${branch}`.

        nb_name (str): Complete name of the notebook file to convert. Can be a
            relative path (eg. './foo/test.ipynb').

    Kwargs:
        template (str): Template to use for conversion. Valid templates:

            - 'full': Outputs a complete standalone HTML page with default
              styling. Automatically loads the nbinteract JS library.
            - 'partial': Outputs an HTML partial that can be embedded in
              another page. Automatically loads the nbinteract JS library but
              has no styling.
            - 'plain': Outputs an HTML partial used to embed in an HTML page
              where the nbinteract JS library is already loaded. Does not load
              JS library or styling

        save_first (bool): If True, saves the currently opened notebook before
            converting nb_name. Used to ensure notebook is written to
            filesystem before starting conversion. Does nothing if not in a
            notebook environment.

    Returns:
        None
    """
    if not os.path.isfile(nb_name):
        raise ValueError(
            "{} isn't a path to a file. Double check your "
            "filename and try again.".format(nb_name)
        )

    if save_first:
        _save_nb(nb_name)

    print('Converting notebook...')
    try:
        check_output(
            ['nbinteract', '--template', template, '--spec', spec, nb_name],
            stderr=STDOUT
        )
    except CalledProcessError as err:
        logging.warning(
            ERROR_MESSAGE.format(
                filename=nb_name, error=str(err.output, 'utf-8')
            )
        )
        return

    html_filename = os.path.splitext(nb_name)[0] + '.html'
    display(Markdown(CONVERT_SUCCESS_MD.format(url=html_filename)))


Instruction:
How would you explain what the following Python 3 function does
def _save_nb(nb_name):
    display(Javascript('IPython.notebook.save_checkpoint();'))
    display(Javascript('IPython.notebook.save_notebook();'))
    print('Saving notebook...', end=' ')

    if _wait_for_save(nb_name):
        print("Saved '{}'.".format(nb_name))
    else:
        logging.warning(
            "Could not save your notebook (timed out waiting for "
            "IPython save). Make sure your notebook is saved "
            "and export again."
        )

Response: Attempts to save notebook. If unsuccessful, shows a warning.


Instruction:
Given the following Python 3 function, write the documentation
def _wait_for_save(nb_name, timeout=5):
    modification_time = os.path.getmtime(nb_name)
    start_time = time.time()
    while time.time() < start_time + timeout:
        if (
            os.path.getmtime(nb_name) > modification_time
            and os.path.getsize(nb_name) > 0
        ):
            return True
        time.sleep(0.2)
    return False

Response: Waits for nb_name to update and returns True if a save was detected and False otherwise.


Instruction:
Explain what the following Python 3 code does
def maybe_call(maybe_fn, kwargs: dict, prefix: str = None) -> 'Any':
    if not callable(maybe_fn):
        return maybe_fn

    args = get_fn_args(maybe_fn, kwargs, prefix=prefix)
    return maybe_fn(**args)

Response: Call maybe_fn with kwargs and return maybe_fn.


Instruction:
Write a Python 3 function for
currying maybe_fn and passes in first_arg. If maybe_fn is not a function returns maybe_fn. Otherwise returns maybe_fn.

Response:
def maybe_curry(maybe_fn, first_arg) -> 'Function | Any':
    """
    If maybe_fn is a function, curries it and passes in first_arg. Otherwise
    returns maybe_fn.
    """
    if not callable(maybe_fn):
        return maybe_fn
    return tz.curry(maybe_fn)(first_arg)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def get_fn_args(fn, kwargs: dict, prefix: str = None):
    all_args = get_all_args(fn)
    required_args = get_required_args(fn)
    fn_kwargs = pick_kwargs(kwargs, all_args, prefix)

    missing_args = [arg for arg in required_args if arg not in fn_kwargs]
    if missing_args:
        raise ValueError(
            'The following args are missing for the function '
            '{}: {}.'.format(fn.__name__, missing_args)
        )

    return fn_kwargs

Response:
Given function and a dict of kwargs return a dict containing only the args
    used by the function.


Instruction:
Here you have a function in Python 3, explain what it does
def get_all_args(fn) -> list:
    sig = inspect.signature(fn)
    return list(sig.parameters)

Response: Returns a list of all arguments for the function fn.


Instruction:
How would you code a function in Python 3 to
return a list of required arguments for the function fn.

Response:
def get_required_args(fn) -> list:
    """
    Returns a list of required arguments for the function fn.

    >>> def foo(x, y, z=100): return x + y + z
    >>> get_required_args(foo)
    ['x', 'y']

    >>> def bar(x, y=100, *args, **kwargs): return x
    >>> get_required_args(bar)
    ['x']
    """
    sig = inspect.signature(fn)
    return [
        name for name, param in sig.parameters.items()
        if param.default == inspect._empty and param.kind not in VAR_ARGS
    ]


Instruction:
Implement a Python 3 function for
giving a dict of kwargs and a list of required_args return a dict containing only the args in required_args.

Response:
def pick_kwargs(kwargs: dict, required_args: list, prefix: str = None):
    """
    Given a dict of kwargs and a list of required_args, return a dict
    containing only the args in required_args.

    If prefix is specified, also search for args that begin with '{prefix}__'.
    Removes prefix in returned dict.

    Raises ValueError if both prefixed and unprefixed arg are given in kwargs.

    >>> from pprint import pprint as p # Use pprint to sort dict keys
    >>> kwargs = {'a': 1, 'b': 2, 'c': 3, 'x__d': 4}
    >>> p(pick_kwargs(kwargs, ['a', 'd']))
    {'a': 1}
    >>> p(pick_kwargs(kwargs, ['a', 'd'], prefix='x'))
    {'a': 1, 'd': 4}

    >>> pick_kwargs({'a': 1, 'x__a': 2}, ['a'], prefix='x')
    Traceback (most recent call last):
    ValueError: Both prefixed and unprefixed args were specified for the
    following parameters: ['a']
    """
    picked = {k: v for k, v in kwargs.items() if k in required_args}

    prefixed = {}
    if prefix:
        prefix = prefix + '__'
        prefixed = {
            _remove_prefix(k, prefix): v
            for k, v in kwargs.items()
            if k.startswith(prefix)
            and _remove_prefix(k, prefix) in required_args
        }

    conflicting_args = [k for k in picked if k in prefixed]
    if conflicting_args:
        raise ValueError(
            'Both prefixed and unprefixed args were specified '
            'for the following parameters: {}'.format(conflicting_args)
        )

    return tz.merge(picked, prefixed)


Instruction:
Explain what the following Python 3 code does
def _update_option_docstring(func, allowed, indent='    ' * 3):
    if not (func.__doc__ and '{options}' in func.__doc__):
        return func

    descriptions = [option + ': ' + option_doc[option] for option in allowed]
    full_desc = ('\n' + indent).join(descriptions)
    func.__doc__ = func.__doc__.format(
        options=options_docstring.format(desc=full_desc)
    )
    return func

Response: Updates the docstring of a function to include the options for the plot.


Instruction:
Here you have a function in Python 3, explain what it does
def use_options(allowed):

    def update_docstring(f):
        _update_option_docstring(f, allowed)

        @functools.wraps(f)
        def check_options(*args, **kwargs):
            options = kwargs.get('options', {})
            not_allowed = [
                option for option in options if option not in allowed
                # Don't validate private options
                and not option.startswith('_')
            ]
            if not_allowed:
                logging.warning(
                    'The following options are not supported by '
                    'this function and will likely result in '
                    'undefined behavior: {}.'.format(not_allowed)
                )

            return f(*args, **kwargs)

        return check_options

    return update_docstring

Response: Decorator that validates options in the passed in alphabetical order.


Instruction:
Here you have a function in Python 3, explain what it does
def hist(hist_function, *, options={}, **interact_params):
    params = {
        'marks': [{
            'sample': _array_or_placeholder(hist_function),
            'bins': _get_option('bins'),
            'normalized': _get_option('normalized'),
            'scales': (
                lambda opts: {'sample': opts['x_sc'], 'count': opts['y_sc']}
            ),
        }],
    }
    fig = options.get('_fig', False) or _create_fig(options=options)
    [hist] = _create_marks(
        fig=fig, marks=[bq.Hist], options=options, params=params
    )
    _add_marks(fig, [hist])

    def wrapped(**interact_params):
        hist.sample = util.maybe_call(hist_function, interact_params)

    controls = widgets.interactive(wrapped, **interact_params)

    return widgets.VBox([controls, fig])

Response: Generates an interactive histogram that allows users to change the parameters of the input hist_function.


Instruction:
Write a Python 3 script for
generating an interactive bar chart that allows users to change the parameters of the inputs x_fn and y_fn.

Response:
def bar(x_fn, y_fn, *, options={}, **interact_params):
    """
    Generates an interactive bar chart that allows users to change the
    parameters of the inputs x_fn and y_fn.

    Args:
        x_fn (Array | (*args -> Array str | Array int | Array float)):
            If array, uses array values for categories of bar chart.

            If function, must take parameters to interact with and return an
            array of strings or numbers. These will become the categories on
            the x-axis of the bar chart.

        y_fn (Array | (Array, *args -> Array int | Array float)):
            If array, uses array values for heights of bars.

            If function, must take in the output of x_fn as its first parameter
            and optionally other parameters to interact with. Must return an
            array of numbers. These will become the heights of the bars on the
            y-axis.

    Kwargs:
        {options}

        interact_params (dict): Keyword arguments in the same format as
            `ipywidgets.interact`. One argument is required for each argument
            of both `x_fn` and `y_fn`. If `x_fn` and `y_fn` have conflicting
            parameter names, prefix the corresponding kwargs with `x__` and
            `y__`.

    Returns:
        VBox with two children: the interactive controls and the figure.

    >>> bar(['a', 'b', 'c'], [4, 7, 10])
    VBox(...)

    >>> def categories(n): return np.arange(n)
    >>> def heights(xs, offset):
    ...     return xs + offset
    >>> bar(categories, heights, n=(0, 10), offset=(1, 10))
    VBox(...)

    >>> def multiply(xs, n):
    ...     return xs * n
    >>> bar(categories, multiply, x__n=(0, 10), y__n=(1, 10))
    VBox(...)
    """
    params = {
        'marks': [{
            'x': _array_or_placeholder(x_fn, PLACEHOLDER_RANGE),
            'y': _array_or_placeholder(y_fn)
        }]
    }
    fig = options.get('_fig', False) or _create_fig(
        x_sc=bq.OrdinalScale, options=options
    )
    [bar] = _create_marks(
        fig=fig, marks=[bq.Bars], options=options, params=params
    )
    _add_marks(fig, [bar])

    def wrapped(**interact_params):
        x_data = util.maybe_call(x_fn, interact_params, prefix='x')
        bar.x = x_data

        y_bound = util.maybe_curry(y_fn, x_data)
        bar.y = util.maybe_call(y_bound, interact_params, prefix='y')

    controls = widgets.interactive(wrapped, **interact_params)

    return widgets.VBox([controls, fig])


Instruction:
Explain what the following Python 3 code does
def scatter_drag(
    x_points: 'Array',
    y_points: 'Array',
    *,
    fig=None,
    show_eqn=True,
    options={}
):
    """
    Generates an interactive scatter plot with the best fit line plotted over
    the points. The points can be dragged by the user and the line will
    automatically update.

    Args:
        x_points (Array Number): x-values of points to plot

        y_points (Array Number): y-values of points to plot

    Kwargs:
        show_eqn (bool): If True (default), displays the best fit line's
            equation above the scatterplot.

        {options}

    Returns:
        VBox with two children: the equation widget and the figure.

    >>> xs = np.arange(10)
    >>> ys = np.arange(10) + np.random.rand(10)
    >>> scatter_drag(xs, ys)
    VBox(...)
    """
    params = {
        'marks': [{
            'x': x_points,
            'y': y_points,
            'enable_move': True,
        }, {
            'colors': [GOLDENROD],
        }]
    }
    fig = options.get('_fig', False) or _create_fig(options=options)
    [scat, lin] = _create_marks(
        fig=fig, marks=[bq.Scatter, bq.Lines], options=options, params=params
    )
    _add_marks(fig, [scat, lin])

    equation = widgets.Label()

    # create line fit to data and display equation
    def update_line(change=None):
        x_sc = scat.scales['x']
        lin.x = [
            x_sc.min if x_sc.min is not None else np.min(scat.x),
            x_sc.max if x_sc.max is not None else np.max(scat.x),
        ]
        poly = np.polyfit(scat.x, scat.y, deg=1)
        lin.y = np.polyval(poly, lin.x)
        if show_eqn:
            equation.value = 'y = {:.2f}x + {:.2f}'.format(poly[0], poly[1])

    update_line()

    scat.observe(update_line, names=['x', 'y'])

    return widgets.VBox([equation, fig])

Response: Generates an interactive scatter plot showing the best fit line of the given points.


Instruction:
Can you generate the documentation for the following Python 3 function
def scatter(x_fn, y_fn, *, options={}, **interact_params):
    params = {
        'marks': [{
            'x': _array_or_placeholder(x_fn),
            'y': _array_or_placeholder(y_fn),
            'marker': _get_option('marker'),
        }]
    }
    fig = options.get('_fig', False) or _create_fig(options=options)
    [scat] = _create_marks(
        fig=fig, marks=[bq.Scatter], options=options, params=params
    )
    _add_marks(fig, [scat])

    def wrapped(**interact_params):
        x_data = util.maybe_call(x_fn, interact_params, prefix='x')
        scat.x = x_data

        y_bound = util.maybe_curry(y_fn, x_data)
        scat.y = util.maybe_call(y_bound, interact_params, prefix='y')

    controls = widgets.interactive(wrapped, **interact_params)

    return widgets.VBox([controls, fig])

Response: Generates an interactive scatter chart that allows users to change the parameters of the inputs x_fn and y_fn.


Instruction:
How would you implement a function in Python 3 that
generates an interactive line chart that allows users to change the parameters of x_fn and y_fn.

Response:
def line(x_fn, y_fn, *, options={}, **interact_params):
    """
    Generates an interactive line chart that allows users to change the
    parameters of the inputs x_fn and y_fn.

    Args:
        x_fn (Array | (*args -> Array str | Array int | Array float)):
            If array, uses array values for x-coordinates.

            If function, must take parameters to interact with and return an
            array of strings or numbers. These will become the x-coordinates
            of the line plot.

        y_fn (Array | (Array, *args -> Array int | Array float)):
            If array, uses array values for y-coordinates.

            If function, must take in the output of x_fn as its first parameter
            and optionally other parameters to interact with. Must return an
            array of numbers. These will become the y-coordinates of the line
            plot.

    Kwargs:
        {options}

        interact_params (dict): Keyword arguments in the same format as
            `ipywidgets.interact`. One argument is required for each argument
            of both `x_fn` and `y_fn`. If `x_fn` and `y_fn` have conflicting
            parameter names, prefix the corresponding kwargs with `x__` and
            `y__`.

    Returns:
        VBox with two children: the interactive controls and the figure.

    >>> line([1, 2, 3], [4, 7, 10])
    VBox(...)

    >>> def x_values(max): return np.arange(0, max)
    >>> def y_values(xs, sd):
    ...     return xs + np.random.normal(len(xs), scale=sd)
    >>> line(x_values, y_values, max=(10, 50), sd=(1, 10))
    VBox(...)
    """
    fig = options.get('_fig', False) or _create_fig(options=options)
    [line] = (_create_marks(fig=fig, marks=[bq.Lines], options=options))
    _add_marks(fig, [line])

    def wrapped(**interact_params):
        x_data = util.maybe_call(x_fn, interact_params, prefix='x')
        line.x = x_data

        y_bound = util.maybe_curry(y_fn, x_data)
        line.y = util.maybe_call(y_bound, interact_params, prefix='y')

    controls = widgets.interactive(wrapped, **interact_params)

    return widgets.VBox([controls, fig])


Instruction:
Create a Python 3 function for
performing a 2 - level deep merge of params with _default_params with corrent merging of params for each mark.

Response:
def _merge_with_defaults(params):
    """
    Performs a 2-level deep merge of params with _default_params with corrent
    merging of params for each mark.

    This is a bit complicated since params['marks'] is a list and we need to
    make sure each mark gets the default params.
    """
    marks_params = [
        tz.merge(default, param) for default, param in
        zip(itertools.repeat(_default_params['marks']), params['marks'])
    ] if 'marks' in params else [_default_params['marks']]

    merged_without_marks = tz.merge_with(
        tz.merge, tz.dissoc(_default_params, 'marks'),
        tz.dissoc(params, 'marks')
    )

    return tz.merge(merged_without_marks, {'marks': marks_params})


Instruction:
How would you code a function in Python 3 to
create a blank figure for a clustered plot.

Response:
def _create_fig(
    *,
    x_sc=bq.LinearScale,
    y_sc=bq.LinearScale,
    x_ax=bq.Axis,
    y_ax=bq.Axis,
    fig=bq.Figure,
    options={},
    params={}
):
    """
    Initializes scales and axes for a bqplot figure and returns the resulting
    blank figure. Each plot component is passed in as a class. The plot options
    should be passed into options.

    Any additional parameters to initialize plot components are passed into
    params as a dict of { plot_component: { trait: value, ... } }

    For example, to change the grid lines of the x-axis:
    params={ 'x_ax': {'grid_lines' : 'solid'} }

    If the param value is a function, it will be called with the options dict
    augmented with all previously created plot elements. This permits
    dependencies on plot elements:
    params={ 'x_ax': {'scale': lambda opts: opts['x_sc'] } }
    """
    params = _merge_with_defaults(params)

    x_sc = x_sc(**_call_params(params['x_sc'], options))
    y_sc = y_sc(**_call_params(params['y_sc'], options))
    options = tz.merge(options, {'x_sc': x_sc, 'y_sc': y_sc})

    x_ax = x_ax(**_call_params(params['x_ax'], options))
    y_ax = y_ax(**_call_params(params['y_ax'], options))
    options = tz.merge(options, {'x_ax': x_ax, 'y_ax': y_ax, 'marks': []})

    fig = fig(**_call_params(params['fig'], options))
    return fig


Instruction:
Write a Python 3 function for
creating a list of mark classes for a figure.

Response:
def _create_marks(fig, marks=[bq.Mark], options={}, params={}):
    """
    Initializes and returns marks for a figure as a list. Each mark is passed
    in as a class. The plot options should be passed into options.

    Any additional parameters to initialize plot components are passed into
    params as a dict of { 'mark': [{ trait: value, ... }, ...] }

    For example, when initializing two marks you can assign different colors to
    each one:

    params={
        'marks': [
            {'colors': [DARK_BLUE]},
            {'colors': [GOLDENROD]},
        ]
    }

    If the param value is a function, it will be called with the options dict
    augmented with all previously created plot elements. This permits
    dependencies on plot elements:
    params={ 'marks': {'scale': lambda opts: opts['x_sc'] } }
    """
    params = _merge_with_defaults(params)

    # Although fig provides scale_x and scale_y properties, the scales on the
    # axes are the only ones that are actually used.
    x_ax, y_ax = fig.axes
    x_sc, y_sc = x_ax.scale, y_ax.scale

    options = tz.merge(options, {'x_sc': x_sc, 'y_sc': y_sc})

    marks = [
        mark_cls(**_call_params(mark_params, options))
        for mark_cls, mark_params in zip(marks, params['marks'])
    ]
    return marks


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _array_or_placeholder(
    maybe_iterable, placeholder=PLACEHOLDER_ZEROS
) -> np.array:
    """
    Return maybe_iterable's contents or a placeholder array.

    Used to give bqplot its required initial points to plot even if we're using
    a function to generate points.
    """
    if isinstance(maybe_iterable, collections.Iterable):
        return np.array([i for i in maybe_iterable])
    return placeholder

Response: Return maybe_iterable s contents or a placeholder array.


Instruction:
How would you explain what the following Python 3 function does
def binder_spec_from_github_url(github_url):
    tokens = re.split(r'/|:', github_url.replace('.git', ''))
    # The username and reponame are the last two tokens
    return '{}/{}/master'.format(tokens[-2], tokens[-1])

Response: Converts a GitHub origin into a Binder spec.


Instruction:
Can you create a Python 3 function that
asks a yes or no question via input and return their answer.

Response:
def yes_or_no(question, default="yes"):
    """Ask a yes/no question via input() and return their answer.

    "question" is a string that is presented to the user.
    "default" is the presumed answer if the user just hits <Enter>.
        It must be "yes" (the default), "no" or None (meaning
        an answer is required of the user).

    The "answer" return value is True for "yes" or False for "no".
    """
    valid = {"yes": True, "y": True, "ye": True, "no": False, "n": False}
    if default is None:
        prompt = " [y/n] "
    elif default == "yes":
        prompt = " [Y/n] "
    elif default == "no":
        prompt = " [y/N] "
    else:
        raise ValueError("invalid default answer: '%s'" % default)

    while True:
        sys.stdout.write(
            '{}[nbinteract]{} {}{}'.format(BLUE, NOCOLOR, question, prompt)
        )
        choice = input().lower()
        if default is not None and choice == '':
            return valid[default]
        elif choice in valid:
            return valid[choice]
        else:
            sys.stdout.write(
                "Please respond with 'yes' or 'no' "
                "(or 'y' or 'n').\n"
            )


Instruction:
Write a Python 3 script for
parsing command line options and runs nbinteract.

Response:
def main():
    """
    Parses command line options and runs nbinteract.
    """
    arguments = docopt(__doc__)
    if arguments['init']:
        return_code = init()
        sys.exit(return_code)

    run_converter(arguments)


Instruction:
Implement a function in Python 3 to
convert notebooks to HTML files.

Response:
def run_converter(arguments):
    """
    Converts notebooks to HTML files. Returns list of output file paths
    """
    # Get spec from config file
    if os.path.isfile(CONFIG_FILE):
        with open(CONFIG_FILE, encoding='utf-8') as f:
            config = json.load(f)
            arguments['--spec'] = arguments['--spec'] or config['spec']

    check_arguments(arguments)

    notebooks = flatmap(
        expand_folder,
        arguments['NOTEBOOKS'],
        recursive=arguments['--recursive']
    )

    exporter = init_exporter(
        extract_images=arguments['--images'],
        spec=arguments['--spec'],
        template_file=arguments['--template'],
        button_at_top=(not arguments['--no-top-button']),
        execute=arguments['--execute'],
    )

    log('Converting notebooks to HTML...')

    output_files = []
    for notebook in notebooks:
        output_file = convert(
            notebook,
            exporter=exporter,
            output_folder=arguments['--output'],
            images_folder=arguments['--images']
        )
        output_files.append(output_file)
        log('Converted {} to {}'.format(notebook, output_file))

    log('Done!')

    if arguments['--images']:
        log('Resulting images located in {}'.format(arguments['--images']))

    return output_files


Instruction:
Write a Python 3 script to
initialize the nbinteract project.

Response:
def init():
    '''
    Initializes git repo for nbinteract.

    1. Checks for requirements.txt or Dockerfile, offering to create a
       requirements.txt if needed.
    2. Sets the Binder spec using the `origin` git remote in .nbinteract.json.
    3. Prints a Binder URL so the user can debug their image if needed.
    '''
    log('Initializing folder for nbinteract.')
    log()

    log('Checking to see if this folder is the root folder of a git project.')
    if os.path.isdir('.git'):
        log("Looks like we're in the root of a git project.")
    else:
        error(
            "This folder doesn't look like the root of a git project. "
            "Please rerun nbinteract init in the top-level folder of a "
            "git project."
        )
        return ERROR
    log()

    log('Checking for requirements.txt or Dockerfile.')
    if os.path.isfile('Dockerfile'):
        log(
            'Dockerfile found. Note that Binder will use the Dockerfile '
            'instead of the requirements.txt file, so you should make sure '
            'your Dockerfile follows the format in {docker_docs}'
            .format(docker_docs=DOCKER_DOCS)
        )
    elif os.path.isfile('requirements.txt'):
        log('requirements.txt found.')
    else:
        log('No requirements.txt file found.')
        yes = yes_or_no(
            'Would you like to create a sample requirements.txt file?'
        )
        if yes:
            # TODO(sam): Don't hard-code requirements.txt
            with open('requirements.txt', 'w', encoding='utf-8') as f:
                f.write(DEFAULT_REQUIREMENTS_TXT)
            log(
                'Created requirements.txt. Edit this file now to include the '
                'rest of your dependencies, then rerun nbinteract init.'
            )
            return SUCCESS
        else:
            log(
                'Please manually create a requirements.txt file, then rerun '
                'nbinteract init.'
            )
            return SUCCESS
    log()

    log('Generating .nbinteract.json file...')
    if os.path.isfile(CONFIG_FILE):
        log(
            ".nbinteract.json already exists, skipping generation. If you'd "
            "like to regenerate the file, remove .nbinteract.json and rerun "
            "this command."
        )
        log()
        log("Initialization success!")
        return SUCCESS

    try:
        github_origin = str(
            subprocess.check_output(
                'git remote get-url origin',
                stderr=subprocess.STDOUT,
                shell=True
            ), 'utf-8'
        ).strip()
    except subprocess.CalledProcessError as e:
        error(
            "No git remote called origin found. Please set up your project's"
            "origin remote to point to a GitHub URL.\ngit error: {}".format(e)
        )
        return ERROR

    if 'github' not in github_origin:
        error(
            "Your project's origin remote {} doesn't look like a github "
            "URL. This may cause issues with Binder, so please double check "
            "your .nbinteract.json file after this script finishes. "
            "Continuing as planned..."
        )

    binder_spec = binder_spec_from_github_url(github_origin)
    with open(CONFIG_FILE, 'w', encoding='utf-8') as f:
        json.dump({'spec': binder_spec}, f, indent=4)
    log('Created .nbinteract.json file successfully')
    log()

    log(
        'Initialization complete! Now, you should make a git commit with the '
        'files created by in this process and push your commits to GitHub.'
    )
    log()
    log(
        'After you push, you should visit {} and verify that your Binder '
        'image successfully starts.'.format(BINDER_BASE_URL + binder_spec)
    )


Instruction:
Create a Python 3 function for
expanding a folder to a list of notebooks.

Response:
def expand_folder(notebook_or_folder, recursive=False):
    """
    If notebook_or_folder is a folder, returns a list containing all notebooks
    in the folder. Otherwise, returns a list containing the notebook name.

    If recursive is True, recurses into subdirectories.
    """
    is_file = os.path.isfile(notebook_or_folder)
    is_dir = os.path.isdir(notebook_or_folder)
    if not (is_file or is_dir):
        raise ValueError(
            '{} is neither an existing file nor a folder.'
            .format(notebook_or_folder)
        )

    if is_file:
        return [notebook_or_folder]

    # Now we know the input is a directory
    if not recursive:
        return glob('{}/*.ipynb'.format(notebook_or_folder))

    # Recursive case
    return [
        os.path.join(folder, filename)
        for folder, _, filenames in os.walk(notebook_or_folder)
        # Skip folders that start with .
        if not os.path.basename(folder).startswith('.')
        for filename in fnmatch.filter(filenames, '*.ipynb')
    ]


Instruction:
Given the following Python 3 function, write the documentation
def init_exporter(extract_images, execute, **exporter_config):
    config = Config(InteractExporter=exporter_config)

    preprocessors = []
    if extract_images:
        # Use ExtractOutputPreprocessor to extract the images to separate files
        preprocessors.append(
            'nbconvert.preprocessors.ExtractOutputPreprocessor'
        )
    if execute:
        # Use the NbiExecutePreprocessor to correctly generate widget output
        # for interact() calls.
        preprocessors.append('nbinteract.preprocessors.NbiExecutePreprocessor')

    config.InteractExporter.preprocessors = preprocessors

    exporter = InteractExporter(config=config)
    return exporter

Response: Returns an initialized exporter.


Instruction:
Create a Python 3 function to
create a dictionary that contains the resources for the exporter.

Response:
def make_exporter_resources(nb_name, out_folder, images_folder=None):
    """
    Creates resources dict for the exporter
    """
    resources = defaultdict(str)
    resources['metadata'] = defaultdict(str)
    resources['metadata']['name'] = nb_name
    resources['metadata']['path'] = out_folder

    # This results in images like AB_5_1.png for a notebook called AB.ipynb
    resources['unique_key'] = nb_name
    resources['output_files_dir'] = images_folder

    return resources


Instruction:
Here you have a function in Python 3, explain what it does
def convert(notebook_path, exporter, output_folder=None, images_folder=None):
    if output_folder:
        os.makedirs(output_folder, exist_ok=True)
    if images_folder:
        os.makedirs(images_folder, exist_ok=True)

    # Computes notebooks/ch1 and <name>.ipynb from notebooks/ch1/<name>.ipynb
    path, filename = os.path.split(notebook_path)
    # Computes <name> from <name>.ipynb
    basename, _ = os.path.splitext(filename)
    # Computes <name>.html from notebooks/<name>.ipynb
    outfile_name = basename + '.html'

    # If output_folder is not set, we default to the original folder of the
    # notebook.
    out_folder = path if not output_folder else output_folder
    outfile_path = os.path.join(out_folder, outfile_name)

    notebook = nbformat.read(notebook_path, as_version=4)

    html, resources = exporter.from_notebook_node(
        notebook,
        resources=make_exporter_resources(basename, out_folder, images_folder),
    )

    # Write out HTML
    with open(outfile_path, 'w', encoding='utf-8') as outfile:
        outfile.write(html)

    # Write out images. If images_folder wasn't specified, resources['outputs']
    # is None so this loop won't run
    for image_path, image_data in resources.get('outputs', {}).items():
        with open(image_path, 'wb') as outimage:
            outimage.write(image_data)

    return outfile_path

Response: Converts a notebook into an HTML file outputting notebooks into output_folder if set and images into images_folder if set.


Instruction:
Can you create a Python 3 function that
converts notebooks in notebook_paths to HTML partials.

Response:
def convert_notebooks_to_html_partial(notebook_paths, url_map):
    """
    Converts notebooks in notebook_paths to HTML partials
    """
    for notebook_path in notebook_paths:
        # Computes <name>.ipynb from notebooks/01/<name>.ipynb
        path, filename = os.path.split(notebook_path)
        # Computes examples from notebooks/examples
        chapter = os.path.split(path)[1] if os.sep in path else ''
        # Computes <name> from <name>.ipynb
        basename, _ = os.path.splitext(filename)
        # Computes <name>.html from notebooks/<name>.ipynb
        outfile_name = basename + '.html'

        # This results in images like AB_5_1.png for a notebook called AB.ipynb
        unique_image_key = basename
        # This sets the img tag URL in the rendered HTML.
        output_files_dir = '/' + NOTEBOOK_IMAGE_DIR

        # Path to output final HTML file
        outfile_path = os.path.join(chapter, outfile_name)
        if chapter:
            os.makedirs(chapter, exist_ok=True)

        extract_output_config = {
            'unique_key': unique_image_key,
            'output_files_dir': output_files_dir,
        }

        notebook = nbformat.read(notebook_path, 4)
        notebook.cells.insert(0, _preamble_cell(path))
        html, resources = html_exporter.from_notebook_node(
            notebook,
            resources=extract_output_config,
        )

        if outfile_path not in url_map:
            print(
                '[Warning]: {} not found in _data/toc.yml. This page will '
                'not appear in the textbook table of contents.'
                .format(outfile_path)
            )
        prev_page = url_map.get(outfile_path, {}).get('prev', 'false')
        next_page = url_map.get(outfile_path, {}).get('next', 'false')

        final_output = wrapper.format(
            html=html,
            prev_page=prev_page,
            next_page=next_page,
        )

        # Write out HTML
        with open(outfile_path, 'w', encoding='utf-8') as outfile:
            outfile.write(final_output)

        # Write out images
        for relative_path, image_data in resources['outputs'].items():
            image_name = os.path.basename(relative_path)
            final_image_path = os.path.join(NOTEBOOK_IMAGE_DIR, image_name)
            with open(final_image_path, 'wb') as outimage:
                outimage.write(image_data)

        print(outfile_path + " written.")


Instruction:
Can you create a Python 3 function that
creates a single notebook cell that will be used in the notebook file.

Response:
def _preamble_cell(path):
    """
    This cell is inserted at the start of each notebook to set the working
    directory to the correct folder.
    """
    code = dedent(
        '''
    # HIDDEN
    # Clear previously defined variables
    %reset -f
    '''.format(path)
    )
    return nbformat.v4.new_code_cell(source=code)


Instruction:
Explain what the following Python 3 code does
def generate_url_map(yaml_path=TOC_PATH) -> dict:
    with open(yaml_path) as f:
        data = yaml.load(f)

    pipeline = [
        t.remove(_not_internal_link),
        flatmap(_flatten_sections),
        t.map(t.get('url')), list, _sliding_three,
        t.map(_adj_pages),
        t.merge()
    ]
    return t.pipe(data, *pipeline)

Response: Generates a dictionary mapping each URL to its previous and next URLs in the textbook.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def find_element_class(self, element_name):

        for element in self.elements:
            if element.element == element_name:
                return element

        return Element

Response: Finds an element class for the given element name contained within the registry. Returns an Element when there is no matching element subclass.


Instruction:
Write a Python 3 function for
transforming data to a new type.

Response:
def _transform(transformer_chain: Sequence[Tuple[DataTransformer, Type]], data: S, context: PipelineContext = None) -> T:
    """Transform data to a new type.

    Args:
        transformer_chain: A sequence of (transformer, type) pairs to convert the data.
        data: The data to be transformed.
        context: The context of the transformations (mutable).

    Returns:
        The transformed data.
    """
    for transformer, target_type in transformer_chain:
        # noinspection PyTypeChecker
        data = transformer.transform(target_type, data, context)
    return data


Instruction:
Can you generate the documentation for the following Python 3 function
def put(self, item: T, context: PipelineContext = None) -> None:
        LOGGER.info("Converting item \"{item}\" for sink \"{sink}\"".format(item=item, sink=self._sink))
        item = self._transform(data=item, context=context)
        LOGGER.info("Puting item \"{item}\" into sink \"{sink}\"".format(item=item, sink=self._sink))
        self._sink.put(self._store_type, item, context)

Response: Puts an object into the data sink.


Instruction:
Write a Python 3 function for
putting multiple objects of the same type into the data sink.

Response:
def put_many(self, items: Iterable[T], context: PipelineContext = None) -> None:
        """Puts multiple objects of the same type into the data sink. The objects may be transformed into a new type for insertion if necessary.

        Args:
            items: An iterable (e.g. list) of objects to be inserted into the data sink.
            context: The context of the insertions (mutable).
        """
        LOGGER.info("Creating transform generator for items \"{items}\" for sink \"{sink}\"".format(items=items, sink=self._sink))
        transform_generator = (self._transform(data=item, context=context) for item in items)
        LOGGER.info("Putting transform generator for items \"{items}\" into sink \"{sink}\"".format(items=items, sink=self._sink))
        self._sink.put_many(self._store_type, transform_generator, context)


Instruction:
Make a summary of the following Python 3 code
def get(self, query: Mapping[str, Any], context: PipelineContext = None) -> T:
        result = self._source.get(self._source_type, deepcopy(query), context)
        LOGGER.info("Got result \"{result}\" from query \"{query}\" of source \"{source}\"".format(result=result, query=query, source=self._source))

        LOGGER.info("Sending result \"{result}\" to sinks before converting".format(result=result))
        for sink in self._before_transform:
            sink.put(result, context)

        LOGGER.info("Converting result \"{result}\" to request type".format(result=result))
        result = self._transform(data=result, context=context)

        LOGGER.info("Sending result \"{result}\" to sinks after converting".format(result=result))
        for sink in self._after_transform:
            sink.put(result, context)

        return result

Response: Gets a query from the data source and transforms it into the requested type.


Instruction:
Here you have a function in Python 3, explain what it does
def get_many(self, query: Mapping[str, Any], context: PipelineContext = None, streaming: bool = False) -> Iterable[T]:
        result = self._source.get_many(self._source_type, deepcopy(query), context)
        LOGGER.info("Got results \"{result}\" from query \"{query}\" of source \"{source}\"".format(result=result, query=query, source=self._source))

        if not streaming:
            LOGGER.info("Non-streaming get_many request. Ensuring results \"{result}\" are a Iterable".format(result=result))
            result = list(result)

            LOGGER.info("Sending results \"{result}\" to sinks before converting".format(result=result))
            for sink in self._before_transform:
                sink.put_many(result, context)

            LOGGER.info("Converting results \"{result}\" to request type".format(result=result))
            result = [self._transform(data=item, context=context) for item in result]

            LOGGER.info("Sending results \"{result}\" to sinks after converting".format(result=result))
            for sink in self._after_transform:
                sink.put_many(result, context)

            return result
        else:
            LOGGER.info("Streaming get_many request. Returning result generator for results \"{result}\"".format(result=result))
            return self._get_many_generator(result)

Response: Gets a query from the data source where the query contains multiple elements to be extracted and transforms the result into the requested type.


Instruction:
Can you write a function in Python 3 where it
gets a query from the data pipeline.

Response:
def get(self, type: Type[T], query: Mapping[str, Any]) -> T:
        """Gets a query from the data pipeline.

        1) Extracts the query the sequence of data sources.
        2) Inserts the result into the data sinks (if appropriate).
        3) Transforms the result into the requested type if it wasn't already.
        4) Inserts the transformed result into any data sinks.

        Args:
            query: The query being requested.
            context: The context for the extraction (mutable).

        Returns:
            The requested object.
        """
        LOGGER.info("Getting SourceHandlers for \"{type}\"".format(type=type.__name__))
        try:
            handlers = self._get_types[type]
        except KeyError:
            try:
                LOGGER.info("Building new SourceHandlers for \"{type}\"".format(type=type.__name__))
                handlers = self._get_handlers(type)
            except NoConversionError:
                handlers = None
            self._get_types[type] = handlers

        if handlers is None:
            raise NoConversionError("No source can provide \"{type}\"".format(type=type.__name__))

        LOGGER.info("Creating new PipelineContext")
        context = self._new_context()

        LOGGER.info("Querying SourceHandlers for \"{type}\"".format(type=type.__name__))
        for handler in handlers:
            try:
                return handler.get(query, context)
            except NotFoundError:
                pass

        raise NotFoundError("No source returned a query result!")


Instruction:
Write a Python 3 script for
getting a query from the data pipeline which contains a request for multiple objects.

Response:
def get_many(self, type: Type[T], query: Mapping[str, Any], streaming: bool = False) -> Iterable[T]:
        """Gets a query from the data pipeline, which contains a request for multiple objects.

        1) Extracts the query the sequence of data sources.
        2) Inserts the results into the data sinks (if appropriate).
        3) Transforms the results into the requested type if it wasn't already.
        4) Inserts the transformed result into any data sinks.

        Args:
            query: The query being requested (contains a request for multiple objects).
            context: The context for the extraction (mutable).
            streaming: Specifies whether the results should be returned as a generator (default False).

        Returns:
            The requested objects or a generator of the objects if streaming is True.
        """
        LOGGER.info("Getting SourceHandlers for \"{type}\"".format(type=type.__name__))
        try:
            handlers = self._get_types[type]
        except KeyError:
            try:
                LOGGER.info("Building new SourceHandlers for \"{type}\"".format(type=type.__name__))
                handlers = self._get_handlers(type)
            except NoConversionError:
                handlers = None
            self._get_types[type] = handlers

        if handlers is None:
            raise NoConversionError("No source can provide \"{type}\"".format(type=type.__name__))

        LOGGER.info("Creating new PipelineContext")
        context = self._new_context()

        LOGGER.info("Querying SourceHandlers for \"{type}\"".format(type=type.__name__))
        for handler in handlers:
            try:
                return handler.get_many(query, context, streaming)
            except NotFoundError:
                pass

        raise NotFoundError("No source returned a query result!")


Instruction:
Given the following Python 3 function, write the documentation
def put(self, type: Type[T], item: T) -> None:
        LOGGER.info("Getting SinkHandlers for \"{type}\"".format(type=type.__name__))
        try:
            handlers = self._put_types[type]
        except KeyError:
            try:
                LOGGER.info("Building new SinkHandlers for \"{type}\"".format(type=type.__name__))
                handlers = self._put_handlers(type)
            except NoConversionError:
                handlers = None
            self._get_types[type] = handlers

        LOGGER.info("Creating new PipelineContext")
        context = self._new_context()

        LOGGER.info("Sending item \"{item}\" to SourceHandlers".format(item=item))
        if handlers is not None:
            for handler in handlers:
                handler.put(item, context)

Response: Puts an object into the data pipeline.


Instruction:
Write a Python 3 script to
put multiple objects of the same type into the data sink.

Response:
def put_many(self, type: Type[T], items: Iterable[T]) -> None:
        """Puts multiple objects of the same type into the data sink. The objects may be transformed into a new type for insertion if necessary.

        Args:
            items: An iterable (e.g. list) of objects to be inserted into the data pipeline.
        """
        LOGGER.info("Getting SinkHandlers for \"{type}\"".format(type=type.__name__))
        try:
            handlers = self._put_types[type]
        except KeyError:
            try:
                LOGGER.info("Building new SinkHandlers for \"{type}\"".format(type=type.__name__))
                handlers = self._put_handlers(type)
            except NoConversionError:
                handlers = None
            self._get_types[type] = handlers

        LOGGER.info("Creating new PipelineContext")
        context = self._new_context()

        LOGGER.info("Sending items \"{items}\" to SourceHandlers".format(items=items))
        if handlers is not None:
            items = list(items)
            for handler in handlers:
                handler.put_many(items, context)


Instruction:
Can you tell what is the following Python 3 function doing
def print_error(self, wrapper):
        level = 0
        parent = wrapper.parent
        while parent:
            print_test_msg(parent.name, level, TestStatus.FAIL, self.use_color)
            level += 1
            parent = parent.parent

        print_test_msg(wrapper.name, level, TestStatus.FAIL, self.use_color)
        print_test_args(wrapper.execute_kwargs, level, TestStatus.FAIL,
                        self.use_color)

        if wrapper.error:
            for line in wrapper.error:
                print_test_msg(
                    line,
                    level + 2,
                    TestStatus.FAIL,
                    self.use_color
                )

        print_expects(wrapper, level, use_color=self.use_color)

Response: Print the error message for the current test.


Instruction:
Can you generate the documentation for the following Python 3 function
def refract(structure) -> Element:

    if isinstance(structure, Element):
        return structure
    elif isinstance(structure, str):
        return String(content=structure)
    elif isinstance(structure, bool):
        return Boolean(content=structure)
    elif isinstance(structure, (int, float)):
        return Number(content=structure)
    elif isinstance(structure, (list, tuple)):
        return Array(content=list(map(refract, structure)))
    elif isinstance(structure, dict):
        return Object(content=[Member(key=refract(k), value=refract(v))
                               for (k, v) in structure.items()])
    elif structure is None:
        return Null()

    raise ValueError('Unsupported Value Type')

Response: Refracts the given value.


Instruction:
How would you explain what the following Python 3 function does
def serialise(self, element: Element, **kwargs) -> str:

        return json.dumps(self.serialise_dict(element), **kwargs)

Response: Serialises the given element into JSON.


Instruction:
Implement a function in Python 3 to
serialise the given element into Compact JSON.

Response:
def serialise(self, element: Element) -> str:
        """
        Serialises the given element into Compact JSON.

        >>> CompactJSONSerialiser().serialise(String(content='Hello'))
        '["string", null, null, "Hello"]'
        """

        return json.dumps(self.serialise_element(element))


Instruction:
Given the following Python 3 function, write the documentation
def deserialise(self, content) -> Element:

        content = json.loads(content)
        if not isinstance(content, list):
            raise ValueError('Given content was not compact JSON refract')

        return self.deserialise_element(content)

Response: Deserialises the given compact JSON into an element.


Instruction:
Can you generate the documentation for the following Python 3 function
def transforms(self) -> Mapping[Type, Iterable[Type]]:
        try:
            return getattr(self.__class__, "transform")._transforms
        except AttributeError:
            return {}

Response: The available data transformers.


Instruction:
Implement a Python 3 function for
transforming an object to a new type.

Response:
def transform(self, target_type: Type[T], value: F, context: PipelineContext = None) -> T:
        """Transforms an object to a new type.

        Args:
            target_type: The type to be converted to.
            value: The object to be transformed.
            context: The context of the transformation (mutable).
        """
        pass


Instruction:
Can you create a Python 3 function that
serializes the CaseWrapper object for collection.

Response:
def serialize(self):
        """ Serializes the CaseWrapper object for collection.

        Warning, this will only grab the available information.
        It is strongly that you only call this once all specs and
        tests have completed.
        """
        expects = [exp.serialize() for exp in self.expects]
        converted_dict = {
            'id': self.id,
            'name': self.pretty_name,
            'raw_name': self.name,
            'doc': self.doc,
            'error': self.error,
            'skipped': self.skipped,
            'skip_reason': self.skip_reason,
            'execute_kwargs': self.safe_execute_kwargs,
            'metadata': self.metadata,
            'start': self.start_time,
            'end': self.end_time,
            'expects': expects,
            'success': self.success
        }
        return remove_empty_entries_from_dict(converted_dict)


Instruction:
Create a Python 3 function to
serialize the Spec / Describe object for collection.

Response:
def serialize(self):
        """ Serializes the Spec/Describe object for collection.

        Warning, this will only grab the available information.
        It is strongly that you only call this once all specs and
        tests have completed.
        """

        cases = [case.serialize() for key, case in six.iteritems(self.cases)]
        specs = [spec.serialize() for spec in self.describes]

        converted_dict = {
            'id': self.id,
            'name': self.name,
            'class_path': self.real_class_path,
            'doc': self.doc,
            'cases': cases,
            'specs': specs
        }
        return converted_dict


Instruction:
Write a Python 3 function that can
call any registered hooks providing the current state.

Response:
def _run_hooks(self):
        """Calls any registered hooks providing the current state."""
        for hook in self.hooks:
            getattr(self, hook)(self._state)


Instruction:
Here you have a function in Python 3, explain what it does
def append(self, element):

        from refract.refraction import refract
        self.content.append(refract(element))

Response: Append an element onto the array.


Instruction:
Write a Python 3 script to
insert an element at a given position.

Response:
def insert(self, index: int, element):
        """
        Insert an element at a given position.

        >>> array = Array()
        >>> array.insert(0, Element())
        """

        from refract.refraction import refract
        self.content.insert(index, refract(element))


Instruction:
Explain what the following Python 3 code does
def index(self, element: Element) -> int:

        from refract.refraction import refract
        return self.content.index(refract(element))

Response: Return the index in the array of the first item whose value is element.


Instruction:
Explain what the following Python 3 code does
def get(self, type: Type[T], query: Mapping[str, Any], context: PipelineContext = None) -> T:
        pass

Response: Gets a query from the data source.


Instruction:
Make a summary of the following Python 3 code
def get_many(self, type: Type[T], query: Mapping[str, Any], context: PipelineContext = None) -> Iterable[T]:
        pass

Response: Gets a query from the data source which contains a request for multiple objects.


Instruction:
Here you have a function in Python 3, explain what it does
def subscribe_all_to_spec(self, spec):
        for reporter in self.reporters:
            if self.can_use_reporter(reporter, self.parallel):
                reporter.subscribe_to_spec(spec)

Response: Subscribe all the available reporters to a spec.


Instruction:
Here you have a function in Python 3, explain what it does
def output(self, msg, indent, status=None):
        color = None
        if self.use_color:
            color = get_color_from_status(status)
        print_indent_msg(msg, indent, color)

Response: Alias for print_indent_msg with color determined by status.


Instruction:
Write a Python 3 function that can
return the assets in the transaction.

Response:
def assets(self) -> List[Asset]:
        """
        Returns the assets in the transaction.
        """

        return list(filter(is_element(Asset), self.content))


Instruction:
Implement a function in Python 3 to
write a list of nodes separated by separator.

Response:
def write_expression_list(self, nodes, separator=', ', allow_newlines=True, need_parens=True,
                              final_separator_if_multiline=True):
        """Writes a list of nodes, separated by separator.

        If allow_newlines, will write the expression over multiple lines if necessary to say within
        max_line_length. If need_parens, will surround the expression with parentheses in this case.
        If final_separator_if_multiline, will write a separator at the end of the list if it is
        divided over multiple lines.

        """
        first = True
        last_line = len(self.lines)
        current_line = list(self.current_line)
        for node in nodes:
            if first:
                first = False
            else:
                self.write(separator)
            self.visit(node)
            if allow_newlines and (self.current_line_length() > self.max_line_length or
                                   last_line != len(self.lines)):
                break
        else:
            return  # stayed within the limit

        # reset state
        del self.lines[last_line:]
        self.current_line = current_line

        separator = separator.rstrip()
        if need_parens:
            self.write('(')
        self.write_newline()
        with self.add_indentation():
            num_nodes = len(nodes)
            for i, node in enumerate(nodes):
                self.write_indentation()
                self.visit(node)
                if final_separator_if_multiline or i < num_nodes - 1:
                    self.write(separator)
                self.write_newline()

        self.write_indentation()
        if need_parens:
            self.write(')')


Instruction:
Write a Python 3 script for
returning the underlying ( unrefracted ) value of element

Response:
def defract(self):
        """
        Returns the underlying (unrefracted) value of element

        >>> Element(content='Hello').defract
        'Hello'

        >>> Element(content=Element(content='Hello')).defract
        'Hello'

        >>> Element(content=[Element(content='Hello')]).defract
        ['Hello']
        """

        from refract.elements.object import Object

        def get_value(item):
            if isinstance(item, KeyValuePair):
                return (get_value(item.key), get_value(item.value))
            elif isinstance(item, list):
                return [get_value(element) for element in item]
            elif isinstance(item, Element):
                if isinstance(item, Object) or item.element == 'object':
                    return dict(get_value(item.content))
                return get_value(item.content)

            return item

        return get_value(self)


Instruction:
Can you create a Python 3 function that
returns all of the children elements.

Response:
def children(self):
        """
        Returns all of the children elements.
        """

        if isinstance(self.content, list):
            return self.content
        elif isinstance(self.content, Element):
            return [self.content]
        else:
            return []


Instruction:
How would you explain what the following Python 3 function does
def recursive_children(self):

        for child in self.children:
            yield child

            for recursive_child in child.recursive_children:
                yield recursive_child

Response: Generator returning all recursive children elements.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def expect(obj, caller_args=[]):
    line, module = get_module_and_line('__spec__')
    src_params = ExpectParams(line, module)

    expect_obj = ExpectAssert(
        obj,
        src_params=src_params,
        caller_args=caller_args
    )
    _add_expect_to_wrapper(expect_obj)
    return expect_obj

Response: This method creates a ExpectAssert object that will be passed to the expect method.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def require(obj, caller_args=[]):
    line, module = get_module_and_line('__spec__')
    src_params = ExpectParams(line, module)

    require_obj = RequireAssert(
        obj,
        src_params=src_params,
        caller_args=caller_args
    )
    _add_expect_to_wrapper(require_obj)
    return require_obj

Response:
This method creates a RequireAssert object that will be used to test assertions in the Specter
   .


Instruction:
How would you explain what the following Python 3 function does
def skip(reason):
    def decorator(test_func):
        if not isinstance(test_func, (type, ClassObjType)):
            func_data = None
            if test_func.__name__ == 'DECORATOR_ONCALL':
                # Call down and save the results
                func_data = test_func()

            @functools.wraps(test_func)
            def skip_wrapper(*args, **kwargs):
                other_data = {
                    'real_func': func_data[0] if func_data else test_func,
                    'metadata': func_data[1] if func_data else None
                }
                raise TestSkippedException(test_func, reason, other_data)
            test_func = skip_wrapper
        return test_func
    return decorator

Response: Decorator for test functions that will always bypass a test.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def incomplete(test_func):
    if not isinstance(test_func, (type, ClassObjType)):
        @functools.wraps(test_func)
        def skip_wrapper(*args, **kwargs):
            raise TestIncompleteException(test_func, _('Test is incomplete'))
        return skip_wrapper

Response: Decorator that makes test_func skip if the test is marked as incomplete.


Instruction:
How would you code a function in Python 3 to
serialize the ExpectAssert object for collection.

Response:
def serialize(self):
        """Serializes the ExpectAssert object for collection.

        Warning, this will only grab the available information.
        It is strongly that you only call this once all specs and
        tests have completed.
        """
        converted_dict = {
            'success': self.success,
            'assertion': str(self),
            'required': self.required
        }
        return converted_dict


Instruction:
Create a Python 3 function to
put multiple objects of the same type into the data sink.

Response:
def put_many(self, type: Type[T], items: Iterable[T], context: PipelineContext = None) -> None:
        """Puts multiple objects of the same type into the data sink.

        Args:
            type: The type of the objects being inserted.
            items: The objects to be inserted.
            context: The context of the insertion (mutable).
        """
        pass


Instruction:
Can you generate a brief explanation for the following Python 3 code
def open(filename, mode="rb", format = None):
    if format == "auto" or format == None:
        format = detect_format(filename)

    if format == "warc":
        return WARCFile(filename, mode)
    elif format == "arc":
        return ARCFile(filename, mode)
    else:
        raise IOError("Don't know how to open '%s' files"%format)

Response: Open a file in the specified mode.


Instruction:
Can you implement a function in Python 3 that
initializes important headers to default values if not already specified.

Response:
def init_defaults(self):
        """Initializes important headers to default values, if not already specified.

        The WARC-Record-ID header is set to a newly generated UUID.
        The WARC-Date header is set to the current datetime.
        The Content-Type is set based on the WARC-Type header.
        The Content-Length is initialized to 0.
        """
        if "WARC-Record-ID" not in self:
            self['WARC-Record-ID'] = "<urn:uuid:%s>" % uuid.uuid1()
        if "WARC-Date" not in self:
            self['WARC-Date'] = datetime.datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ')
        if "Content-Type" not in self:
            self['Content-Type'] = WARCHeader.CONTENT_TYPES.get(self.type, "application/octet-stream")


Instruction:
Here you have a function in Python 3, explain what it does
def write_to(self, f):
        f.write(self.version + "\r\n")
        for name, value in self.items():
            name = name.title()
            # Use standard forms for commonly used patterns
            name = name.replace("Warc-", "WARC-").replace("-Ip-", "-IP-").replace("-Id", "-ID").replace("-Uri", "-URI")
            f.write(name)
            f.write(": ")
            f.write(value)
            f.write("\r\n")

        # Header ends with an extra CRLF
        f.write("\r\n")

Response: Writes this header to a file f.


Instruction:
How would you explain what the following Python 3 function does
def from_response(response):
        # Get the httplib.HTTPResponse object
        http_response = response.raw._original_response

        # HTTP status line, headers and body as strings
        status_line = "HTTP/1.1 %d %s" % (http_response.status, http_response.reason)
        headers = str(http_response.msg)
        body = http_response.read()

        # Monkey-patch the response object so that it is possible to read from it later.
        response.raw._fp = StringIO(body)

        # Build the payload to create warc file.
        payload = status_line + "\r\n" + headers + "\r\n" + body

        headers = {
            "WARC-Type": "response",
            "WARC-Target-URI": response.request.full_url.encode('utf-8')
        }
        return WARCRecord(payload=payload, headers=headers)

Response: Creates a WARCRecord object from a given httplib. HTTPResponse object.


Instruction:
How would you implement a function in Python 3 that
adds a warc record to this WARC file.

Response:
def write_record(self, warc_record):
        """Adds a warc record to this WARC file.
        """
        warc_record.write_to(self.fileobj)
        # Each warc record is written as separate member in the gzip file
        # so that each record can be read independetly.
        if isinstance(self.fileobj, gzip2.GzipFile):
            self.fileobj.close_member()


Instruction:
Make a summary of the following Python 3 code
def tell(self):
        if isinstance(self.fileobj, gzip2.GzipFile):
            return self.fileobj.fileobj.tell()
        else:
            return self.fileobj.tell()

Response: Returns the file offset.


Instruction:
Write a Python 3 function for
closing the current member being written.

Response:
def close_member(self):
        """Closes the current member being written.
        """
        # The new member is not yet started, no need to close
        if self._new_member:
            return

        self.fileobj.write(self.compress.flush())
        write32u(self.fileobj, self.crc)
        # self.size may exceed 2GB, or even 4GB
        write32u(self.fileobj, self.size & 0xffffffffL)
        self.size = 0
        self.compress = zlib.compressobj(9,
                                         zlib.DEFLATED,
                                         -zlib.MAX_WBITS,
                                         zlib.DEF_MEM_LEVEL,
                                         0)
        self._new_member = True


Instruction:
Implement a Python 3 function for
starting writing a new member if required.

Response:
def _start_member(self):
        """Starts writing a new member if required.
        """
        if self._new_member:
            self._init_write(self.name)
            self._write_gzip_header()
            self._new_member = False


Instruction:
Here you have a function in Python 3, explain what it does
def close(self):
        if self.fileobj is None:
            return
        if self.mode == WRITE:
            self.close_member()
            self.fileobj = None
        elif self.mode == READ:
            self.fileobj = None

        if self.myfileobj:
            self.myfileobj.close()
            self.myfileobj = None

Response: Closes the gzip with care to handle multiple members.


Instruction:
Can you generate the documentation for the following Python 3 function
def read_member(self):
        if self._member_lock is False:
            self._member_lock = True

        if self._new_member:
            try:
                # Read one byte to move to the next member
                BaseGzipFile._read(self, 1)
                assert self._new_member is False
            except EOFError:
                return None

        return self

Response: Returns a file - like object to read one member from the gzip file.


Instruction:
Can you implement a function in Python 3 that
writes the given data as one gzip member.

Response:
def write_member(self, data):
        """Writes the given data as one gzip member.

        The data can be a string, an iterator that gives strings or a file-like object.
        """
        if isinstance(data, basestring):
            self.write(data)
        else:
            for text in data:
                self.write(text)
        self.close_member()


Instruction:
How would you explain what the following Python 3 function does
def write_to(self, f, version = None):
        if not version:
            version = self.version
        if version == 1:
            header = "%(url)s %(ip_address)s %(date)s %(content_type)s %(length)s"
        elif version == 2:
            header = "%(url)s %(ip_address)s %(date)s %(content_type)s %(result_code)s %(checksum)s %(location)s %(offset)s %(filename)s %(length)s"

        header =  header%dict(url          = self['url'],
                              ip_address   = self['ip_address'],
                              date         = self['date'],
                              content_type = self['content_type'],
                              result_code  = self['result_code'],
                              checksum     = self['checksum'],
                              location     = self['location'],
                              offset       = self['offset'],
                              filename     = self['filename'],
                              length       = self['length'])
        f.write(header)

Response: Writes out the arc header to the file like object f.


Instruction:
Can you write a function in Python 3 where it
constructs an ARC record from a string and returns it.

Response:
def from_string(cls, string, version):
        """
        Constructs an ARC record from a string and returns it.

        TODO: It might be best to merge this with the _read_arc_record
        function rather than reimplement the functionality here.
        """
        header, payload = string.split("\n",1)
        if payload[0] == '\n': # There's an extra
            payload = payload[1:]
        if int(version) == 1:
            arc_header_re = ARC1_HEADER_RE
        elif int(version) == 2:
            arc_header_re = ARC2_HEADER_RE

        matches = arc_header_re.search(header)
        headers = matches.groupdict()
        arc_header = ARCHeader(**headers)
        return cls(header = arc_header, payload = payload, version = version)


Instruction:
Create a Python 3 function for
writing out an ARC header

Response:
def _write_header(self):
        "Writes out an ARC header"
        if "org" not in self.file_headers:
            warnings.warn("Using 'unknown' for Archiving organisation name")
            self.file_headers['org'] = "Unknown"
        if "date" not in self.file_headers:
            now = datetime.datetime.utcnow()
            warnings.warn("Using '%s' for Archiving time"%now)
            self.file_headers['date'] = now
        if "ip_address" not in self.file_headers:
            warnings.warn("Using '127.0.0.1' as IP address of machine that's archiving")
            self.file_headers['ip_address'] = "127.0.0.1"
        if self.version == 1:
            payload = "1 0 %(org)s\nURL IP-address Archive-date Content-type Archive-length"%dict(org = self.file_headers['org'])
        elif self.version == 2:
            payload = "2 0 %(org)s\nURL IP-address Archive-date Content-type Result-code Checksum Location Offset Filename Archive-length"
        else:
            raise IOError("Can't write an ARC file with version '\"%s\"'"%self.version)

        fname = os.path.basename(self.fileobj.name)
        header = ARCHeader(url = "filedesc://%s"%fname,
                           ip_address = self.file_headers['ip_address'],
                           date = self.file_headers['date'],
                           content_type = "text/plain",
                           length = len(payload),
                           result_code = "200",
                           checksum = "-",
                           location = "-",
                           offset = str(self.fileobj.tell()),
                           filename = fname)
        arc_file_header_record = ARCRecord(header, payload%self.file_headers)
        self.write(arc_file_header_record)


Instruction:
Explain what the following Python 3 code does
def write(self, arc_record):
        "Writes out the given arc record to the file"
        if not self.version:
            self.version = 2
        if not self.header_written:
            self.header_written = True
            self._write_header()
        arc_record.write_to(self.fileobj, self.version)
        self.fileobj.write("\n")

Response: Writes out the given arc record to the file