Instruction:
Can you generate the documentation for the following Python 3 function
def cli(self, *args, **kwargs):
        kwargs['api'] = self.api
        return cli(*args, **kwargs)

Response: Defines a function that should be routed by this API


Instruction:
Can you write a function in Python 3 where it
registers a class based router to this API

Response:
def object(self, *args, **kwargs):
        """Registers a class based router to this API"""
        kwargs['api'] = self.api
        return Object(*args, **kwargs)


Instruction:
Create a Python 3 function for
returning a generator of all URLs attached to this API

Response:
def urls(self):
        """Returns a generator of all URLs attached to this API"""
        for base_url, mapping in self.routes.items():
            for url, _ in mapping.items():
                yield base_url + url


Instruction:
How would you code a function in Python 3 to
return all registered handlers attached to this API

Response:
def handlers(self):
        """Returns all registered handlers attached to this API"""
        used = []
        for base_url, mapping in self.routes.items():
            for url, methods in mapping.items():
                for method, versions in methods.items():
                    for version, handler in versions.items():
                        if not handler in used:
                            used.append(handler)
                            yield handler


Instruction:
Can you implement a function in Python 3 that
returns the set input_format handler for the given content_type

Response:
def input_format(self, content_type):
        """Returns the set input_format handler for the given content_type"""
        return getattr(self, '_input_format', {}).get(content_type, hug.defaults.input_format.get(content_type, None))


Instruction:
How would you implement a function in Python 3 that
sets an input format handler for this Hug API given the specified content_type

Response:
def set_input_format(self, content_type, handler):
        """Sets an input format handler for this Hug API, given the specified content_type"""
        if getattr(self, '_input_format', None) is None:
            self._input_format = {}
        self._input_format[content_type] = handler


Instruction:
Write a Python 3 function that can
add a middleware object used to process all incoming requests against the API.

Response:
def add_middleware(self, middleware):
        """Adds a middleware object used to process all incoming requests against the API"""
        if self.middleware is None:
            self._middleware = []
        self.middleware.append(middleware)


Instruction:
Write a Python 3 function for
adding an exception handler to the hug api

Response:
def add_exception_handler(self, exception_type, error_handler, versions=(None, )):
        """Adds a error handler to the hug api"""
        versions = (versions, ) if not isinstance(versions, (tuple, list)) else versions
        if not hasattr(self, '_exception_handlers'):
            self._exception_handlers = {}

        for version in versions:
            placement = self._exception_handlers.setdefault(version, OrderedDict())
            placement[exception_type] = (error_handler, ) + placement.get(exception_type, tuple())


Instruction:
Implement a Python 3 function for
extending this instance with the given http_api.

Response:
def extend(self, http_api, route="", base_url="", **kwargs):
        """Adds handlers from a different Hug API to this one - to create a single API"""
        self.versions.update(http_api.versions)
        base_url = base_url or self.base_url

        for router_base_url, routes in http_api.routes.items():
            self.routes.setdefault(base_url, OrderedDict())
            for item_route, handler in routes.items():
                for method, versions in handler.items():
                    for version, function in versions.items():
                        function.interface.api = self.api
                self.routes[base_url].setdefault(route + item_route, {}).update(handler)

        for sink_base_url, sinks in http_api.sinks.items():
            for url, sink in sinks.items():
                self.add_sink(sink, route + url, base_url=base_url)

        for middleware in (http_api.middleware or ()):
            self.add_middleware(middleware)

        for version, handler in getattr(http_api, '_exception_handlers', {}).items():
            for exception_type, exception_handlers in handler.items():
                target_exception_handlers = self.exception_handlers(version) or {}
                for exception_handler in exception_handlers:
                    if exception_type not in target_exception_handlers:
                        self.add_exception_handler(exception_type, exception_handler, version)

        for input_format, input_format_handler in getattr(http_api, '_input_format', {}).items():
            if not input_format in getattr(self, '_input_format', {}):
                self.set_input_format(input_format, input_format_handler)

        for version, handler in http_api.not_found_handlers.items():
            if version not in self.not_found_handlers:
                self.set_not_found_handler(handler, version)


Instruction:
Explain what the following Python 3 code does
def set_not_found_handler(self, handler, version=None):
        if not self.not_found_handlers:
            self._not_found_handlers = {}

        self.not_found_handlers[version] = handler

Response: Sets the not_found handler for the specified version of the api


Instruction:
Here you have a function in Python 3, explain what it does
def documentation(self, base_url=None, api_version=None, prefix=""):
        documentation = OrderedDict()
        base_url = self.base_url if base_url is None else base_url
        overview = self.api.doc
        if overview:
            documentation['overview'] = overview

        version_dict = OrderedDict()
        versions = self.versions
        versions_list = list(versions)
        if None in versions_list:
            versions_list.remove(None)
        if False in versions_list:
            versions_list.remove(False)
        if api_version is None and len(versions_list) > 0:
            api_version = max(versions_list)
            documentation['version'] = api_version
        elif api_version is not None:
            documentation['version'] = api_version
        if versions_list:
            documentation['versions'] = versions_list
        for router_base_url, routes in self.routes.items():
            for url, methods in routes.items():
                for method, method_versions in methods.items():
                    for version, handler in method_versions.items():
                        if getattr(handler, 'private', False):
                            continue
                        if version is None:
                            applies_to = versions
                        else:
                            applies_to = (version, )
                        for version in applies_to:
                            if api_version and version != api_version:
                                continue
                            if base_url and router_base_url != base_url:
                                continue
                            doc = version_dict.setdefault(url, OrderedDict())
                            doc[method] = handler.documentation(doc.get(method, None), version=version, prefix=prefix,
                                                                base_url=router_base_url, url=url)
        documentation['handlers'] = version_dict
        return documentation

Response: Generates and returns documentation for this API endpoint


Instruction:
Create a Python 3 function for
running the basic hug development server against this API

Response:
def serve(self, host='', port=8000, no_documentation=False, display_intro=True):
        """Runs the basic hug development server against this API"""
        if no_documentation:
            api = self.server(None)
        else:
            api = self.server()

        if display_intro:
            print(INTRO)

        httpd = make_server(host, port, api)
        print("Serving on {0}:{1}...".format(host, port))
        httpd.serve_forever()


Instruction:
Implement a function in Python 3 to
determine the appropriate version given the set api_version the request header and URL query params

Response:
def determine_version(self, request, api_version=None):
        """Determines the appropriate version given the set api_version, the request header, and URL query params"""
        if api_version is False:
            api_version = None
            for version in self.versions:
                if version and "v{0}".format(version) in request.path:
                    api_version = version
                    break

        request_version = set()
        if api_version is not None:
            request_version.add(api_version)

        version_header = request.get_header("X-API-VERSION")
        if version_header:
            request_version.add(version_header)

        version_param = request.get_param('api_version')
        if version_param is not None:
            request_version.add(version_param)

        if len(request_version) > 1:
            raise ValueError('You are requesting conflicting versions')

        return next(iter(request_version or (None, )))


Instruction:
Can you generate the documentation for the following Python 3 function
def documentation_404(self, base_url=None):
        base_url = self.base_url if base_url is None else base_url

        def handle_404(request, response, *args, **kwargs):
            url_prefix = request.forwarded_uri[:-1]
            if request.path and request.path != "/":
                url_prefix = request.forwarded_uri.split(request.path)[0]

            to_return = OrderedDict()
            to_return['404'] = ("The API call you tried to make was not defined. "
                                "Here's a definition of the API to help you get going :)")
            to_return['documentation'] = self.documentation(base_url, self.determine_version(request, False),
                                                            prefix=url_prefix)

            if self.output_format == hug.output_format.json:
                response.data = hug.output_format.json(to_return, indent=4, separators=(',', ': '))
                response.content_type = 'application/json; charset=utf-8'
            else:
                response.data = self.output_format(to_return, request=request, response=response)
                response.content_type = self.output_format.content_type

            response.status = falcon.HTTP_NOT_FOUND

        handle_404.interface = True
        return handle_404

Response: Returns a smart 404 page that contains documentation for the written API


Instruction:
Can you tell what is the following Python 3 function doing
def version_router(self, request, response, api_version=None, versions={}, not_found=None, **kwargs):
        request_version = self.determine_version(request, api_version)
        if request_version:
            request_version = int(request_version)
        versions.get(request_version or False, versions.get(None, not_found))(request, response,
                                                                              api_version=api_version,
                                                                              **kwargs)

Response: Intelligently routes a request to the correct handler based on the version being requested


Instruction:
Write a Python 3 script for
returning a WSGI compatible API server for the given Hug API module.

Response:
def server(self, default_not_found=True, base_url=None):
        """Returns a WSGI compatible API server for the given Hug API module"""
        falcon_api = falcon.API(middleware=self.middleware)
        default_not_found = self.documentation_404() if default_not_found is True else None
        base_url = self.base_url if base_url is None else base_url

        not_found_handler = default_not_found
        self.api._ensure_started()
        if self.not_found_handlers:
            if len(self.not_found_handlers) == 1 and None in self.not_found_handlers:
                not_found_handler = self.not_found_handlers[None]
            else:
                not_found_handler = partial(self.version_router, api_version=False,
                                            versions=self.not_found_handlers, not_found=default_not_found)
                not_found_handler.interface = True

        if not_found_handler:
            falcon_api.add_sink(not_found_handler)
            self._not_found = not_found_handler

        for sink_base_url, sinks in self.sinks.items():
            for url, extra_sink in sinks.items():
                falcon_api.add_sink(extra_sink, sink_base_url + url + '(?P<path>.*)')

        for router_base_url, routes in self.routes.items():
            for url, methods in routes.items():
                router = {}
                for method, versions in methods.items():
                    method_function = "on_{0}".format(method.lower())
                    if len(versions) == 1 and None in versions.keys():
                        router[method_function] = versions[None]
                    else:
                        router[method_function] = partial(self.version_router, versions=versions,
                                                          not_found=not_found_handler)

                router = namedtuple('Router', router.keys())(**router)
                falcon_api.add_route(router_base_url + url, router)
                if self.versions and self.versions != (None, ):
                    falcon_api.add_route(router_base_url + '/v{api_version}' + url, router)

        def error_serializer(request, response, error):
            response.content_type = self.output_format.content_type
            response.body = self.output_format({"errors": {error.title: error.description}},
                                               request, response)

        falcon_api.set_error_serializer(error_serializer)
        return falcon_api


Instruction:
Here you have a function in Python 3, explain what it does
def extend(self, cli_api, command_prefix="", sub_command="", **kwargs):
        if sub_command and command_prefix:
            raise ValueError('It is not currently supported to provide both a command_prefix and sub_command')

        if sub_command:
            self.commands[sub_command] = cli_api
        else:
            for name, command in cli_api.commands.items():
                self.commands["{}{}".format(command_prefix, name)] = command

Response: Extends this CLI api with the commands present in the provided CLI api object.


Instruction:
Create a Python 3 function for
returning all directives applicable to this Hug API

Response:
def directives(self):
        """Returns all directives applicable to this Hug API"""
        directive_sources = chain(hug.defaults.directives.items(), getattr(self, '_directives', {}).items())
        return {'hug_' + directive_name: directive for directive_name, directive in directive_sources}


Instruction:
Write a Python 3 function that can
return the loaded directive with the specified name or default if it is not present.

Response:
def directive(self, name, default=None):
        """Returns the loaded directive with the specified name, or default if passed name is not present"""
        return getattr(self, '_directives', {}).get(name,  hug.defaults.directives.get(name, default))


Instruction:
Explain what the following Python 3 code does
def handlers(self):
        if getattr(self, '_http'):
            yield from self.http.handlers()
        if getattr(self, '_cli'):
            yield from self.cli.handlers()

Response: Returns all registered handlers attached to this API


Instruction:
Create a Python 3 function to
add handlers from a different Hug API to this one - to create a single API

Response:
def extend(self, api, route="", base_url="", http=True, cli=True, **kwargs):
        """Adds handlers from a different Hug API to this one - to create a single API"""
        api = API(api)

        if http and hasattr(api, '_http'):
            self.http.extend(api.http, route, base_url, **kwargs)

        if cli and hasattr(api, '_cli'):
            self.cli.extend(api.cli, **kwargs)

        for directive in getattr(api, '_directives', {}).values():
            self.add_directive(directive)

        for startup_handler in (api.startup_handlers or ()):
            self.add_startup_handler(startup_handler)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def add_startup_handler(self, handler):
        if not self.startup_handlers:
            self._startup_handlers = []

        self.startup_handlers.append(handler)

Response: Adds a startup handler to the hug api


Instruction:
Can you create a Python 3 function that
marks the API as started and runs all startup handlers

Response:
def _ensure_started(self):
        """Marks the API as started and runs all startup handlers"""
        if not self.started:
            async_handlers = [startup_handler for startup_handler in self.startup_handlers if
                              introspect.is_coroutine(startup_handler)]
            if async_handlers:
                loop = asyncio.get_event_loop()
                loop.run_until_complete(asyncio.gather(*[handler(self) for handler in async_handlers], loop=loop))
            for startup_handler in self.startup_handlers:
                if not startup_handler in async_handlers:
                    startup_handler(self)


Instruction:
Implement a function in Python 3 to
load the IMDB dataset for binary sentiment classification.

Response:
def imdb_dataset(directory='data/',
                 train=False,
                 test=False,
                 train_directory='train',
                 test_directory='test',
                 extracted_name='aclImdb',
                 check_files=['aclImdb/README'],
                 url='http://ai.stanford.edu/~amaas/data/sentiment/aclImdb_v1.tar.gz',
                 sentiments=['pos', 'neg']):
    """
    Load the IMDB dataset (Large Movie Review Dataset v1.0).

    This is a dataset for binary sentiment classification containing substantially more data than
    previous benchmark datasets. Provided a set of 25,000 highly polar movie reviews for
    training, and 25,000 for testing. There is additional unlabeled data for use as well. Raw text
    and already processed bag of words formats are provided.

    Note:
        The order examples are returned is not guaranteed due to ``iglob``.

    **Reference:** http://ai.stanford.edu/~amaas/data/sentiment/

    Args:
        directory (str, optional): Directory to cache the dataset.
        train (bool, optional): If to load the training split of the dataset.
        test (bool, optional): If to load the test split of the dataset.
        train_directory (str, optional): The directory of the training split.
        test_directory (str, optional): The directory of the test split.
        extracted_name (str, optional): Name of the extracted dataset directory.
        check_files (str, optional): Check if these files exist, then this download was successful.
        url (str, optional): URL of the dataset ``tar.gz`` file.
        sentiments (list of str, optional): Sentiments to load from the dataset.

    Returns:
        :class:`tuple` of :class:`torchnlp.datasets.Dataset` or :class:`torchnlp.datasets.Dataset`:
        Returns between one and all dataset splits (train, dev and test) depending on if their
        respective boolean argument is ``True``.

    Example:
        >>> from torchnlp.datasets import imdb_dataset  # doctest: +SKIP
        >>> train = imdb_dataset(train=True)  # doctest: +SKIP
        >>> train[0:2]  # doctest: +SKIP
        [{
          'text': 'For a movie that gets no respect there sure are a lot of memorable quotes...',
          'sentiment': 'pos'
        }, {
          'text': 'Bizarre horror movie filled with famous faces but stolen by Cristina Raines...',
          'sentiment': 'pos'
        }]
    """
    download_file_maybe_extract(url=url, directory=directory, check_files=check_files)

    ret = []
    splits = [
        dir_ for (requested, dir_) in [(train, train_directory), (test, test_directory)]
        if requested
    ]
    for split_directory in splits:
        full_path = os.path.join(directory, extracted_name, split_directory)
        examples = []
        for sentiment in sentiments:
            for filename in glob.iglob(os.path.join(full_path, sentiment, '*.txt')):
                with open(filename, 'r', encoding="utf-8") as f:
                    text = f.readline()
                examples.append({
                    'text': text,
                    'sentiment': sentiment,
                })
        ret.append(Dataset(examples))

    if len(ret) == 1:
        return ret[0]
    else:
        return tuple(ret)


Instruction:
Here you have a function in Python 3, explain what it does
def trec_dataset(directory='data/trec/',
                 train=False,
                 test=False,
                 train_filename='train_5500.label',
                 test_filename='TREC_10.label',
                 check_files=['train_5500.label'],
                 urls=[
                     'http://cogcomp.org/Data/QA/QC/train_5500.label',
                     'http://cogcomp.org/Data/QA/QC/TREC_10.label'
                 ],
                 fine_grained=False):
    """
    Load the Text REtrieval Conference (TREC) Question Classification dataset.

    TREC dataset contains 5500 labeled questions in training set and another 500 for test set. The
    dataset has 6 labels, 50 level-2 labels. Average length of each sentence is 10, vocabulary size
    of 8700.

    References:
        * https://nlp.stanford.edu/courses/cs224n/2004/may-steinberg-project.pdf
        * http://cogcomp.org/Data/QA/QC/
        * http://www.aclweb.org/anthology/C02-1150

    **Citation:**
    Xin Li, Dan Roth, Learning Question Classifiers. COLING'02, Aug., 2002.

    Args:
        directory (str, optional): Directory to cache the dataset.
        train (bool, optional): If to load the training split of the dataset.
        test (bool, optional): If to load the test split of the dataset.
        train_filename (str, optional): The filename of the training split.
        test_filename (str, optional): The filename of the test split.
        check_files (str, optional): Check if these files exist, then this download was successful.
        urls (str, optional): URLs to download.

    Returns:
        :class:`tuple` of :class:`torchnlp.datasets.Dataset` or :class:`torchnlp.datasets.Dataset`:
        Returns between one and all dataset splits (train, dev and test) depending on if their
        respective boolean argument is ``True``.

    Example:
        >>> from torchnlp.datasets import trec_dataset  # doctest: +SKIP
        >>> train = trec_dataset(train=True)  # doctest: +SKIP
        >>> train[:2]  # doctest: +SKIP
        [{
          'label': 'DESC',
          'text': 'How did serfdom develop in and then leave Russia ?'
        }, {
          'label': 'ENTY',
          'text': 'What films featured the character Popeye Doyle ?'
        }]
    """
    download_files_maybe_extract(urls=urls, directory=directory, check_files=check_files)

    ret = []
    splits = [(train, train_filename), (test, test_filename)]
    splits = [f for (requested, f) in splits if requested]
    for filename in splits:
        full_path = os.path.join(directory, filename)
        examples = []
        for line in open(full_path, 'rb'):
            # there is one non-ASCII byte: sisterBADBYTEcity; replaced with space
            label, _, text = line.replace(b'\xf0', b' ').strip().decode().partition(' ')
            label, _, label_fine = label.partition(':')
            if fine_grained:
                examples.append({'label': label_fine, 'text': text})
            else:
                examples.append({'label': label, 'text': text})
        ret.append(Dataset(examples))

    if len(ret) == 1:
        return ret[0]
    else:
        return tuple(ret)

Response: Load the TREC dataset.


Instruction:
Implement a function in Python 3 to
load the Stanford Natural Language Inference dataset.

Response:
def snli_dataset(directory='data/',
                 train=False,
                 dev=False,
                 test=False,
                 train_filename='snli_1.0_train.jsonl',
                 dev_filename='snli_1.0_dev.jsonl',
                 test_filename='snli_1.0_test.jsonl',
                 extracted_name='snli_1.0',
                 check_files=['snli_1.0/snli_1.0_train.jsonl'],
                 url='http://nlp.stanford.edu/projects/snli/snli_1.0.zip'):
    """
    Load the Stanford Natural Language Inference (SNLI) dataset.

    The SNLI corpus (version 1.0) is a collection of 570k human-written English sentence pairs
    manually labeled for balanced classification with the labels entailment, contradiction, and
    neutral, supporting the task of natural language inference (NLI), also known as recognizing
    textual entailment (RTE). We aim for it to serve both as a benchmark for evaluating
    representational systems for text, especially including those induced by representation
    learning methods, as well as a resource for developing NLP models of any kind.

    **Reference:** https://nlp.stanford.edu/projects/snli/

    **Citation:**
    Samuel R. Bowman, Gabor Angeli, Christopher Potts, and Christopher D. Manning. 2015. A large
    annotated corpus for learning natural language inference. In Proceedings of the 2015 Conference
    on Empirical Methods in Natural Language Processing (EMNLP).

    Args:
        directory (str, optional): Directory to cache the dataset.
        train (bool, optional): If to load the training split of the dataset.
        dev (bool, optional): If to load the development split of the dataset.
        test (bool, optional): If to load the test split of the dataset.
        train_filename (str, optional): The filename of the training split.
        dev_filename (str, optional): The filename of the development split.
        test_filename (str, optional): The filename of the test split.
        extracted_name (str, optional): Name of the extracted dataset directory.
        check_files (str, optional): Check if these files exist, then this download was successful.
        url (str, optional): URL of the dataset `tar.gz` file.

    Returns:
        :class:`tuple` of :class:`torchnlp.datasets.Dataset` or :class:`torchnlp.datasets.Dataset`:
        Returns between one and all dataset splits (train, dev and test) depending on if their
        respective boolean argument is ``True``.

    Example:
        >>> from torchnlp.datasets import snli_dataset  # doctest: +SKIP
        >>> train = snli_dataset(train=True)  # doctest: +SKIP
        >>> train[0]  # doctest: +SKIP
        {
          'premise': 'Kids are on a amusement ride.',
          'hypothesis': 'A car is broke down on the side of the road.',
          'label': 'contradiction',
          'premise_transitions': ['shift', 'shift', 'shift', 'shift', 'shift', 'shift', ...],
          'hypothesis_transitions': ['shift', 'shift', 'shift', 'shift', 'shift', 'shift', ...],
        }
    """
    download_file_maybe_extract(url=url, directory=directory, check_files=check_files)

    get_transitions = lambda parse: ['reduce' if t == ')' else 'shift' for t in parse if t != '(']
    ret = []
    splits = [(train, train_filename), (dev, dev_filename), (test, test_filename)]
    splits = [f for (requested, f) in splits if requested]
    for filename in splits:
        full_path = os.path.join(directory, extracted_name, filename)
        examples = []
        with io.open(full_path, encoding='utf-8') as f:
            for line in f:
                line = line.strip()
                line = json.loads(line)
                examples.append({
                    'premise': line['sentence1'],
                    'hypothesis': line['sentence2'],
                    'label': line['gold_label'],
                    'premise_transitions': get_transitions(line['sentence1_binary_parse']),
                    'hypothesis_transitions': get_transitions(line['sentence2_binary_parse'])
                })
        ret.append(Dataset(examples))

    if len(ret) == 1:
        return ret[0]
    else:
        return tuple(ret)


Instruction:
Can you implement a function in Python 3 that
forwards the attention mechanism to the next entry in the sequence of entries in the context.

Response:
def forward(self, query, context):
        """
        Args:
            query (:class:`torch.FloatTensor` [batch size, output length, dimensions]): Sequence of
                queries to query the context.
            context (:class:`torch.FloatTensor` [batch size, query length, dimensions]): Data
                overwhich to apply the attention mechanism.

        Returns:
            :class:`tuple` with `output` and `weights`:
            * **output** (:class:`torch.LongTensor` [batch size, output length, dimensions]):
              Tensor containing the attended features.
            * **weights** (:class:`torch.FloatTensor` [batch size, output length, query length]):
              Tensor containing attention weights.
        """
        batch_size, output_len, dimensions = query.size()
        query_len = context.size(1)

        if self.attention_type == "general":
            query = query.view(batch_size * output_len, dimensions)
            query = self.linear_in(query)
            query = query.view(batch_size, output_len, dimensions)

        # TODO: Include mask on PADDING_INDEX?

        # (batch_size, output_len, dimensions) * (batch_size, query_len, dimensions) ->
        # (batch_size, output_len, query_len)
        attention_scores = torch.bmm(query, context.transpose(1, 2).contiguous())

        # Compute weights across every context sequence
        attention_scores = attention_scores.view(batch_size * output_len, query_len)
        attention_weights = self.softmax(attention_scores)
        attention_weights = attention_weights.view(batch_size, output_len, query_len)

        # (batch_size, output_len, query_len) * (batch_size, query_len, dimensions) ->
        # (batch_size, output_len, dimensions)
        mix = torch.bmm(attention_weights, context)

        # concat -> (batch_size * output_len, 2*dimensions)
        combined = torch.cat((mix, query), dim=2)
        combined = combined.view(batch_size * output_len, 2 * dimensions)

        # Apply linear_out on every 2nd dimension of concat
        # output -> (batch_size, output_len, dimensions)
        output = self.linear_out(combined).view(batch_size, output_len, dimensions)
        output = self.tanh(output)

        return output, attention_weights


Instruction:
Write a Python 3 script for
loading the IWSLT 2017 translation dataset.

Response:
def iwslt_dataset(
        directory='data/iwslt/',
        train=False,
        dev=False,
        test=False,
        language_extensions=['en', 'de'],
        train_filename='{source}-{target}/train.{source}-{target}.{lang}',
        dev_filename='{source}-{target}/IWSLT16.TED.tst2013.{source}-{target}.{lang}',
        test_filename='{source}-{target}/IWSLT16.TED.tst2014.{source}-{target}.{lang}',
        check_files=['{source}-{target}/train.tags.{source}-{target}.{source}'],
        url='https://wit3.fbk.eu/archive/2016-01/texts/{source}/{target}/{source}-{target}.tgz'):
    """
    Load the International Workshop on Spoken Language Translation (IWSLT) 2017 translation dataset.

    In-domain training, development and evaluation sets were supplied through the website of the
    WIT3 project, while out-of-domain training data were linked in the workshop’s website. With
    respect to edition 2016 of the evaluation campaign, some of the talks added to the TED
    repository during the last year have been used to define the evaluation sets (tst2017), while
    the remaining new talks have been included in the training sets.

    The English data that participants were asked to recognize and translate consists in part of
    TED talks as in the years before, and in part of real-life lectures and talks that have been
    mainly recorded in lecture halls at KIT and Carnegie Mellon University. TED talks are
    challenging due to their variety in topics, but are very benign as they are very thoroughly
    rehearsed and planned, leading to easy to recognize and translate language.

    Note:
        The order examples are returned is not guaranteed due to ``iglob``.

    References:
      * http://workshop2017.iwslt.org/downloads/iwslt2017_proceeding_v2.pdf
      * http://workshop2017.iwslt.org/

    **Citation:**
    M. Cettolo, C. Girardi, and M. Federico. 2012. WIT3: Web Inventory of Transcribed and Translated
    Talks. In Proc. of EAMT, pp. 261-268, Trento, Italy.

    Args:
        directory (str, optional): Directory to cache the dataset.
        train (bool, optional): If to load the training split of the dataset.
        dev (bool, optional): If to load the dev split of the dataset.
        test (bool, optional): If to load the test split of the dataset.
        language_extensions (:class:`list` of :class:`str`): Two language extensions
            ['en'|'de'|'it'|'ni'|'ro'] to load.
        train_filename (str, optional): The filename of the training split.
        dev_filename (str, optional): The filename of the dev split.
        test_filename (str, optional): The filename of the test split.
        check_files (str, optional): Check if these files exist, then this download was successful.
        url (str, optional): URL of the dataset file.

    Returns:
        :class:`tuple` of :class:`torchnlp.datasets.Dataset` or :class:`torchnlp.datasets.Dataset`:
        Returns between one and all dataset splits (train, dev and test) depending on if their
        respective boolean argument is ``True``.

    Example:
        >>> from torchnlp.datasets import iwslt_dataset  # doctest: +SKIP
        >>> train = iwslt_dataset(train=True)  # doctest: +SKIP
        >>> train[:2]  # doctest: +SKIP
        [{
          'en': "David Gallo: This is Bill Lange. I'm Dave Gallo.",
          'de': 'David Gallo: Das ist Bill Lange. Ich bin Dave Gallo.'
        }, {
          'en': "And we're going to tell you some stories from the sea here in video.",
          'de': 'Wir werden Ihnen einige Geschichten über das Meer in Videoform erzählen.'
        }]
    """
    if len(language_extensions) != 2:
        raise ValueError("`language_extensions` must be two language extensions "
                         "['en'|'de'|'it'|'ni'|'ro'] to load.")

    # Format Filenames
    source, target = tuple(language_extensions)
    check_files = [s.format(source=source, target=target) for s in check_files]
    url = url.format(source=source, target=target)

    download_file_maybe_extract(url=url, directory=directory, check_files=check_files)

    iwslt_clean(os.path.join(directory, '{source}-{target}'.format(source=source, target=target)))

    ret = []
    splits = [(train, train_filename), (dev, dev_filename), (test, test_filename)]
    splits = [f for (requested, f) in splits if requested]
    for filename in splits:
        examples = []
        for extension in language_extensions:
            path = os.path.join(directory,
                                filename.format(lang=extension, source=source, target=target))
            with open(path, 'r', encoding='utf-8') as f:
                language_specific_examples = [l.strip() for l in f]

            if len(examples) == 0:
                examples = [{} for _ in range(len(language_specific_examples))]
            for i, example in enumerate(language_specific_examples):
                examples[i][extension] = example

        ret.append(Dataset(examples))

    if len(ret) == 1:
        return ret[0]
    else:
        return tuple(ret)


Instruction:
Here you have a function in Python 3, explain what it does
def encode(self, object_):
        if self.enforce_reversible:
            self.enforce_reversible = False
            if self.decode(self.encode(object_)) != object_:
                raise ValueError('Encoding is not reversible for "%s"' % object_)
            self.enforce_reversible = True

        return object_

Response: Encodes an object into a base64 - encoded version of the object.


Instruction:
How would you code a function in Python 3 to
encode a list of objects into a single list of bytes.

Response:
def batch_encode(self, iterator, *args, **kwargs):
        """
        Args:
            batch (list): Batch of objects to encode.
            *args: Arguments passed to ``encode``.
            **kwargs: Keyword arguments passed to ``encode``.

        Returns:
            list: Batch of encoded objects.
        """
        return [self.encode(object_, *args, **kwargs) for object_ in iterator]


Instruction:
Implement a function in Python 3 to
decode an object. Args: object_ (object): Encoded object. Returns: object: Object decoded.

Response:
def decode(self, encoded):
        """ Decodes an object.

        Args:
            object_ (object): Encoded object.

        Returns:
            object: Object decoded.
        """
        if self.enforce_reversible:
            self.enforce_reversible = False
            if self.encode(self.decode(encoded)) != encoded:
                raise ValueError('Decoding is not reversible for "%s"' % encoded)
            self.enforce_reversible = True

        return encoded


Instruction:
Implement a Python 3 function for
decoding a list of encoded objects.

Response:
def batch_decode(self, iterator, *args, **kwargs):
        """
        Args:
            iterator (list): Batch of encoded objects.
            *args: Arguments passed to ``decode``.
            **kwargs: Keyword arguments passed to ``decode``.

        Returns:
            list: Batch of decoded objects.
        """
        return [self.decode(encoded, *args, **kwargs) for encoded in iterator]


Instruction:
Write a Python 3 function that can
forward computation of the sequence.

Response:
def forward(self, tokens, mask=None):
        """
        Args:
            tokens (:class:`torch.FloatTensor` [batch_size, num_tokens, input_dim]): Sequence
                matrix to encode.
            mask (:class:`torch.FloatTensor`): Broadcastable matrix to `tokens` used as a mask.
        Returns:
            (:class:`torch.FloatTensor` [batch_size, output_dim]): Encoding of sequence.
        """
        if mask is not None:
            tokens = tokens * mask.unsqueeze(-1).float()

        # Our input is expected to have shape `(batch_size, num_tokens, embedding_dim)`.  The
        # convolution layers expect input of shape `(batch_size, in_channels, sequence_length)`,
        # where the conv layer `in_channels` is our `embedding_dim`.  We thus need to transpose the
        # tensor first.
        tokens = torch.transpose(tokens, 1, 2)
        # Each convolution layer returns output of size `(batch_size, num_filters, pool_length)`,
        # where `pool_length = num_tokens - ngram_size + 1`.  We then do an activation function,
        # then do max pooling over each filter for the whole input sequence.  Because our max
        # pooling is simple, we just use `torch.max`.  The resultant tensor of has shape
        # `(batch_size, num_conv_layers * num_filters)`, which then gets projected using the
        # projection layer, if requested.

        filter_outputs = []
        for i in range(len(self._convolution_layers)):
            convolution_layer = getattr(self, 'conv_layer_{}'.format(i))
            filter_outputs.append(self._activation(convolution_layer(tokens)).max(dim=2)[0])

        # Now we have a list of `num_conv_layers` tensors of shape `(batch_size, num_filters)`.
        # Concatenating them gives us a tensor of shape
        # `(batch_size, num_filters * num_conv_layers)`.
        maxpool_output = torch.cat(
            filter_outputs, dim=1) if len(filter_outputs) > 1 else filter_outputs[0]

        if self.projection_layer:
            result = self.projection_layer(maxpool_output)
        else:
            result = maxpool_output
        return result


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _unescape_token(escaped_token):

    def match(m):
        if m.group(1) is None:
            return u"_" if m.group(0) == u"\\u" else u"\\"

        try:
            return six.unichr(int(m.group(1)))
        except (ValueError, OverflowError):
            return ""

    trimmed = escaped_token[:-1] if escaped_token.endswith("_") else escaped_token
    return _UNESCAPE_REGEX.sub(match, trimmed)

Response: Unescapes a unicode string.


Instruction:
How would you code a function in Python 3 to
convert a list of tokens to a list of subtoken strings.

Response:
def _tokens_to_subtoken(self, tokens):
        """ Converts a list of tokens to a list of subtoken.

        Args:
          tokens: a list of strings.
        Returns:
          a list of integers in the range [0, vocab_size)
        """
        ret = []
        for token in tokens:
            ret.extend(
                self._escaped_token_to_subtoken_strings(_escape_token(token, self._alphabet)))
        return ret


Instruction:
How would you code a function in Python 3 to
convert a list of subtoken to a list of tokens.

Response:
def _subtoken_to_tokens(self, subtokens):
        """ Converts a list of subtoken to a list of tokens.

        Args:
          subtokens: a list of integers in the range [0, vocab_size)

        Returns:
          a list of strings.
        """
        concatenated = "".join(subtokens)
        split = concatenated.split("_")
        return [_unescape_token(t + "_") for t in split if t]


Instruction:
Given the following Python 3 function, write the documentation
def _escaped_token_to_subtoken_strings(self, escaped_token):
        # NOTE: This algorithm is greedy; it won't necessarily produce the "best"
        # list of subtokens.
        ret = []
        start = 0
        token_len = len(escaped_token)
        while start < token_len:
            for end in xrange(min(token_len, start + self._max_subtoken_len), start, -1):
                subtoken = escaped_token[start:end]
                if subtoken in self._all_subtoken_strings:
                    ret.append(subtoken)
                    start = end
                    break

            else:  # Did not break
                # If there is no possible encoding of the escaped token then one of the
                # characters in the token is not in the alphabet. This should be
                # impossible and would be indicative of a bug.
                assert False, "Token substring not found in subtoken vocabulary."

        return ret

Response: Converts an escaped token string to a list of subtoken strings.


Instruction:
Implement a function in Python 3 to
build a SubwordTextTokenizer instance that has vocab_size near target_size.

Response:
def build_to_target_size_from_token_counts(cls,
                                               target_size,
                                               token_counts,
                                               min_val,
                                               max_val,
                                               num_iterations=4):
        """Builds a SubwordTextTokenizer that has `vocab_size` near `target_size`.

        Uses simple recursive binary search to find a minimum token count that most
        closely matches the `target_size`.

        Args:
          target_size: Desired vocab_size to approximate.
          token_counts: A dictionary of token counts, mapping string to int.
          min_val: An integer; lower bound for the minimum token count.
          max_val: An integer; upper bound for the minimum token count.
          num_iterations: An integer; how many iterations of refinement.

        Returns:
          A SubwordTextTokenizer instance.

        Raises:
          ValueError: If `min_val` is greater than `max_val`.
        """
        if min_val > max_val:
            raise ValueError("Lower bound for the minimum token count "
                             "is greater than the upper bound.")

        def bisect(min_val, max_val):
            """Bisection to find the right size."""
            present_count = (max_val + min_val) // 2
            logger.info("Trying min_count %d" % present_count)
            subtokenizer = cls()
            subtokenizer.build_from_token_counts(token_counts, present_count, num_iterations)
            logger.info("min_count %d attained a %d vocab_size", present_count,
                        subtokenizer.vocab_size)

            # If min_val == max_val, we can't do any better than this.
            if subtokenizer.vocab_size == target_size or min_val >= max_val:
                return subtokenizer

            if subtokenizer.vocab_size > target_size:
                other_subtokenizer = bisect(present_count + 1, max_val)
            else:
                other_subtokenizer = bisect(min_val, present_count - 1)

            if other_subtokenizer is None:
                return subtokenizer

            if (abs(other_subtokenizer.vocab_size - target_size) <
                    abs(subtokenizer.vocab_size - target_size)):
                return other_subtokenizer
            return subtokenizer

        return bisect(min_val, max_val)


Instruction:
How would you code a function in Python 3 to
train a SubwordTextTokenizer based on a dictionary of word counts.

Response:
def build_from_token_counts(self, token_counts, min_count, num_iterations=4):
        """Train a SubwordTextTokenizer based on a dictionary of word counts.

        Args:
          token_counts: a dictionary of Unicode strings to int.
          min_count: an integer - discard subtokens with lower counts.
          num_iterations: an integer; how many iterations of refinement.
        """
        self._init_alphabet_from_tokens(six.iterkeys(token_counts))

        # Bootstrap the initial list of subtokens with the characters from the
        # alphabet plus the escaping characters.
        self._init_subtokens_from_list(list(self._alphabet))

        # We build iteratively.  On each iteration, we segment all the words,
        # then count the resulting potential subtokens, keeping the ones
        # with high enough counts for our new vocabulary.
        if min_count < 1:
            min_count = 1
        for i in xrange(num_iterations):

            # Collect all substrings of the encoded token that break along current
            # subtoken boundaries.
            subtoken_counts = collections.defaultdict(int)
            for token, count in six.iteritems(token_counts):
                escaped_token = _escape_token(token, self._alphabet)
                subtokens = self._escaped_token_to_subtoken_strings(escaped_token)
                start = 0
                for subtoken in subtokens:
                    for end in xrange(start + 1, len(escaped_token) + 1):
                        new_subtoken = escaped_token[start:end]
                        subtoken_counts[new_subtoken] += count
                    start += len(subtoken)

            # Array of sets of candidate subtoken strings, by length.
            len_to_subtoken_strings = []
            for subtoken_string, count in six.iteritems(subtoken_counts):
                lsub = len(subtoken_string)
                if count >= min_count:
                    while len(len_to_subtoken_strings) <= lsub:
                        len_to_subtoken_strings.append(set())
                    len_to_subtoken_strings[lsub].add(subtoken_string)

            # Consider the candidates longest to shortest, so that if we accept
            # a longer subtoken string, we can decrement the counts of its
            # prefixes.
            new_subtoken_strings = []
            for lsub in xrange(len(len_to_subtoken_strings) - 1, 0, -1):
                subtoken_strings = len_to_subtoken_strings[lsub]
                for subtoken_string in subtoken_strings:
                    count = subtoken_counts[subtoken_string]
                    if count >= min_count:
                        # Exclude alphabet tokens here, as they must be included later,
                        # explicitly, regardless of count.
                        if subtoken_string not in self._alphabet:
                            new_subtoken_strings.append((count, subtoken_string))
                        for l in xrange(1, lsub):
                            subtoken_counts[subtoken_string[:l]] -= count

            # Include the alphabet explicitly to guarantee all strings are
            # encodable.
            new_subtoken_strings.extend((subtoken_counts.get(a, 0), a) for a in self._alphabet)
            new_subtoken_strings.sort(reverse=True)

            # Reinitialize to the candidate vocabulary.
            self._init_subtokens_from_list([subtoken for _, subtoken in new_subtoken_strings])


Instruction:
Can you tell what is the following Python 3 function doing
def _init_subtokens_from_list(self, subtoken_strings):
        # we remember the maximum length of any subtoken to avoid having to
        # check arbitrarily long strings.
        self._all_subtoken_strings = set([s for s in subtoken_strings if s])
        self._max_subtoken_len = max([len(s) for s in subtoken_strings])

Response: Initialize token information from a list of subtoken strings.


Instruction:
Explain what the following Python 3 code does
def _init_alphabet_from_tokens(self, tokens):
        # Include all characters from all tokens in the alphabet to guarantee that
        # any token can be encoded. Additionally, include all escaping
        # characters.
        self._alphabet = {c for token in tokens for c in token}
        self._alphabet |= _ESCAPE_CHARS

Response: Initialize the alphabet from an iterable of token or subtoken strings.


Instruction:
Implement a Python 3 function for
getting the BLEU score using the MOSES multi - BLEU script.

Response:
def get_moses_multi_bleu(hypotheses, references, lowercase=False):
    """Get the BLEU score using the moses `multi-bleu.perl` script.

    **Script:**
    https://raw.githubusercontent.com/moses-smt/mosesdecoder/master/scripts/generic/multi-bleu.perl

    Args:
      hypotheses (list of str): List of predicted values
      references (list of str): List of target values
      lowercase (bool): If true, pass the "-lc" flag to the `multi-bleu.perl` script

    Returns:
      (:class:`np.float32`) The BLEU score as a float32 value.

    Example:

      >>> hypotheses = [
      ...   "The brown fox jumps over the dog 笑",
      ...   "The brown fox jumps over the dog 2 笑"
      ... ]
      >>> references = [
      ...   "The quick brown fox jumps over the lazy dog 笑",
      ...   "The quick brown fox jumps over the lazy dog 笑"
      ... ]
      >>> get_moses_multi_bleu(hypotheses, references, lowercase=True)
      46.51
    """
    if isinstance(hypotheses, list):
        hypotheses = np.array(hypotheses)
    if isinstance(references, list):
        references = np.array(references)

    if np.size(hypotheses) == 0:
        return np.float32(0.0)

    # Get MOSES multi-bleu script
    try:
        multi_bleu_path, _ = urllib.request.urlretrieve(
            "https://raw.githubusercontent.com/moses-smt/mosesdecoder/"
            "master/scripts/generic/multi-bleu.perl")
        os.chmod(multi_bleu_path, 0o755)
    except:
        logger.warning("Unable to fetch multi-bleu.perl script")
        return None

    # Dump hypotheses and references to tempfiles
    hypothesis_file = tempfile.NamedTemporaryFile()
    hypothesis_file.write("\n".join(hypotheses).encode("utf-8"))
    hypothesis_file.write(b"\n")
    hypothesis_file.flush()
    reference_file = tempfile.NamedTemporaryFile()
    reference_file.write("\n".join(references).encode("utf-8"))
    reference_file.write(b"\n")
    reference_file.flush()

    # Calculate BLEU using multi-bleu script
    with open(hypothesis_file.name, "r") as read_pred:
        bleu_cmd = [multi_bleu_path]
        if lowercase:
            bleu_cmd += ["-lc"]
        bleu_cmd += [reference_file.name]
        try:
            bleu_out = subprocess.check_output(bleu_cmd, stdin=read_pred, stderr=subprocess.STDOUT)
            bleu_out = bleu_out.decode("utf-8")
            bleu_score = re.search(r"BLEU = (.+?),", bleu_out).group(1)
            bleu_score = float(bleu_score)
            bleu_score = np.float32(bleu_score)
        except subprocess.CalledProcessError as error:
            if error.output is not None:
                logger.warning("multi-bleu.perl script returned non-zero exit code")
                logger.warning(error.output)
            bleu_score = None

    # Close temp files
    hypothesis_file.close()
    reference_file.close()

    return bleu_score


Instruction:
Can you create a Python 3 function that
loads the Universal Dependencies Treebank dataset.

Response:
def ud_pos_dataset(directory='data/',
                   train=False,
                   dev=False,
                   test=False,
                   train_filename='en-ud-tag.v2.train.txt',
                   dev_filename='en-ud-tag.v2.dev.txt',
                   test_filename='en-ud-tag.v2.test.txt',
                   extracted_name='en-ud-v2',
                   check_files=['en-ud-v2/en-ud-tag.v2.train.txt'],
                   url='https://bitbucket.org/sivareddyg/public/downloads/en-ud-v2.zip'):
    """
    Load the Universal Dependencies - English Dependency Treebank dataset.

    Corpus of sentences annotated using Universal Dependencies annotation. The corpus comprises
    254,830 words and 16,622 sentences, taken from various web media including weblogs, newsgroups,
    emails, reviews, and Yahoo! answers.

    References:
        * http://universaldependencies.org/
        * https://github.com/UniversalDependencies/UD_English

    **Citation:**
    Natalia Silveira and Timothy Dozat and Marie-Catherine de Marneffe and Samuel Bowman and
    Miriam Connor and John Bauer and Christopher D. Manning (2014).
    A Gold Standard Dependency Corpus for {E}nglish

    Args:
        directory (str, optional): Directory to cache the dataset.
        train (bool, optional): If to load the training split of the dataset.
        dev (bool, optional): If to load the development split of the dataset.
        test (bool, optional): If to load the test split of the dataset.
        train_filename (str, optional): The filename of the training split.
        dev_filename (str, optional): The filename of the development split.
        test_filename (str, optional): The filename of the test split.
        extracted_name (str, optional): Name of the extracted dataset directory.
        check_files (str, optional): Check if these files exist, then this download was successful.
        url (str, optional): URL of the dataset `tar.gz` file.

    Returns:
        :class:`tuple` of :class:`torchnlp.datasets.Dataset` or :class:`torchnlp.datasets.Dataset`:
        Returns between one and all dataset splits (train, dev and test) depending on if their
        respective boolean argument is ``True``.

    Example:
        >>> from torchnlp.datasets import ud_pos_dataset  # doctest: +SKIP
        >>> train = ud_pos_dataset(train=True)  # doctest: +SKIP
        >>> train[17]  # doctest: +SKIP
        {
          'tokens': ['Guerrillas', 'killed', 'an', 'engineer', ',', 'Asi', 'Ali', ',', 'from',
                     'Tikrit', '.'],
          'ud_tags': ['NOUN', 'VERB', 'DET', 'NOUN', 'PUNCT', 'PROPN', 'PROPN', 'PUNCT', 'ADP',
                      'PROPN', 'PUNCT'],
          'ptb_tags': ['NNS', 'VBD', 'DT', 'NN', ',', 'NNP', 'NNP', ',', 'IN', 'NNP', '.']
        }
    """
    download_file_maybe_extract(url=url, directory=directory, check_files=check_files)

    ret = []
    splits = [(train, train_filename), (dev, dev_filename), (test, test_filename)]
    splits = [f for (requested, f) in splits if requested]
    for filename in splits:
        full_path = os.path.join(directory, extracted_name, filename)
        examples = []
        with io.open(full_path, encoding='utf-8') as f:
            sentence = {'tokens': [], 'ud_tags': [], 'ptb_tags': []}
            for line in f:
                line = line.strip()
                if line == '' and len(sentence['tokens']) > 0:
                    examples.append(sentence)
                    sentence = {'tokens': [], 'ud_tags': [], 'ptb_tags': []}
                elif line != '':
                    token, ud_tag, ptb_tag = tuple(line.split('\t'))
                    sentence['tokens'].append(token)
                    sentence['ud_tags'].append(ud_tag)
                    sentence['ptb_tags'].append(ptb_tag)
        ret.append(Dataset(examples))

    if len(ret) == 1:
        return ret[0]
    else:
        return tuple(ret)


Instruction:
Explain what the following Python 3 code does
def makedirs(name):

    import os, errno

    try:
        os.makedirs(name)
    except OSError as ex:
        if ex.errno == errno.EEXIST and os.path.isdir(name):
            # ignore existing directory
            pass
        else:
            # a different error happened
            raise

Response: helper function for python 2 and 3 to call os. makedirs


Instruction:
Here you have a function in Python 3, explain what it does
def collate_fn(batch, train=True):
    premise_batch, _ = pad_batch([row['premise'] for row in batch])
    hypothesis_batch, _ = pad_batch([row['hypothesis'] for row in batch])
    label_batch = torch.stack([row['label'] for row in batch])

    # PyTorch RNN requires batches to be transposed for speed and integration with CUDA
    transpose = (lambda b: b.t_().squeeze(0).contiguous())

    return (transpose(premise_batch), transpose(hypothesis_batch), transpose(label_batch))

Response: Returns a list of tensors to a batch tensors


Instruction:
Make a summary of the following Python 3 code
def smt_dataset(directory='data/',
                train=False,
                dev=False,
                test=False,
                train_filename='train.txt',
                dev_filename='dev.txt',
                test_filename='test.txt',
                extracted_name='trees',
                check_files=['trees/train.txt'],
                url='http://nlp.stanford.edu/sentiment/trainDevTestTrees_PTB.zip',
                fine_grained=False,
                subtrees=False):
    """
    Load the Stanford Sentiment Treebank dataset.

    Semantic word spaces have been very useful but cannot express the meaning of longer phrases in
    a principled way. Further progress towards understanding compositionality in tasks such as
    sentiment detection requires richer supervised training and evaluation resources and more
    powerful models of composition. To remedy this, we introduce a Sentiment Treebank. It includes
    fine grained sentiment labels for 215,154 phrases in the parse trees of 11,855 sentences and
    presents new challenges for sentiment compositionality.

    **Reference**:
    https://nlp.stanford.edu/sentiment/index.html

    **Citation:**
    Richard Socher, Alex Perelygin, Jean Y. Wu, Jason Chuang, Christopher D. Manning,
    Andrew Y. Ng and Christopher Potts. Recursive Deep Models for Semantic Compositionality Over a
    Sentiment Treebank

    Args:
        directory (str, optional): Directory to cache the dataset.
        train (bool, optional): If to load the training split of the dataset.
        dev (bool, optional): If to load the development split of the dataset.
        test (bool, optional): If to load the test split of the dataset.
        train_filename (str, optional): The filename of the training split.
        dev_filename (str, optional): The filename of the development split.
        test_filename (str, optional): The filename of the test split.
        extracted_name (str, optional): Name of the extracted dataset directory.
        check_files (str, optional): Check if these files exist, then this download was successful.
        url (str, optional): URL of the dataset `tar.gz` file.
        subtrees (bool, optional): Whether to include sentiment-tagged subphrases in addition to
            complete examples.
        fine_grained (bool, optional): Whether to use 5-class instead of 3-class labeling.

    Returns:
        :class:`tuple` of :class:`torchnlp.datasets.Dataset`: Tuple with the training tokens, dev
        tokens and test tokens in order if their respective boolean argument is true.

    Example:
        >>> from torchnlp.datasets import smt_dataset  # doctest: +SKIP
        >>> train = smt_dataset(train=True)  # doctest: +SKIP
        >>> train[5]  # doctest: +SKIP
        {
          'text': "Whether or not you 're enlightened by any of Derrida 's lectures on ...",
          'label': 'positive'
        }
    """
    download_file_maybe_extract(url=url, directory=directory, check_files=check_files)

    ret = []
    splits = [(train, train_filename), (dev, dev_filename), (test, test_filename)]
    splits = [f for (requested, f) in splits if requested]
    for filename in splits:
        full_path = os.path.join(directory, extracted_name, filename)
        examples = []
        with io.open(full_path, encoding='utf-8') as f:
            for line in f:
                line = line.strip()
                if subtrees:
                    examples.extend(parse_tree(line, subtrees=subtrees, fine_grained=fine_grained))
                else:
                    examples.append(parse_tree(line, subtrees=subtrees, fine_grained=fine_grained))
        ret.append(Dataset(examples))

    if len(ret) == 1:
        return ret[0]
    else:
        return tuple(ret)

Response: Loads the Sentiment Treebank dataset and returns a dict of Sentiment Treebank objects.


Instruction:
Write a Python 3 script for
loading the SimpleQuestions dataset.

Response:
def simple_qa_dataset(
        directory='data/',
        train=False,
        dev=False,
        test=False,
        extracted_name='SimpleQuestions_v2',
        train_filename='annotated_fb_data_train.txt',
        dev_filename='annotated_fb_data_valid.txt',
        test_filename='annotated_fb_data_test.txt',
        check_files=['SimpleQuestions_v2/annotated_fb_data_train.txt'],
        url='https://www.dropbox.com/s/tohrsllcfy7rch4/SimpleQuestions_v2.tgz?raw=1',
):  # pragma: no cover
    """
    Load the SimpleQuestions dataset.

    Single-relation factoid questions (simple questions) are common in many settings
    (e.g. Microsoft’s search query logs and WikiAnswers questions). The SimpleQuestions dataset is
    one of the most commonly used benchmarks for studying single-relation factoid questions.

    **Reference:**
    https://research.fb.com/publications/large-scale-simple-question-answering-with-memory-networks/

    Args:
        directory (str, optional): Directory to cache the dataset.
        train (bool, optional): If to load the training split of the dataset.
        dev (bool, optional): If to load the development split of the dataset.
        test (bool, optional): If to load the test split of the dataset.
        extracted_name (str, optional): Name of the extracted dataset directory.
        train_filename (str, optional): The filename of the training split.
        dev_filename (str, optional): The filename of the development split.
        test_filename (str, optional): The filename of the test split.
        check_files (str, optional): Check if these files exist, then this download was successful.
        url (str, optional): URL of the dataset `tar.gz` file.

    Returns:
        :class:`tuple` of :class:`torchnlp.datasets.Dataset` or :class:`torchnlp.datasets.Dataset`:
        Returns between one and all dataset splits (train, dev and test) depending on if their
        respective boolean argument is ``True``.

    Example:
        >>> from torchnlp.datasets import simple_qa_dataset  # doctest: +SKIP
        >>> train = simple_qa_dataset(train=True)  # doctest: +SKIP
        SimpleQuestions_v2.tgz:  15%|▏| 62.3M/423M [00:09<00:41, 8.76MB/s]
        >>> train[0:2]  # doctest: +SKIP
        [{
          'question': 'what is the book e about',
          'relation': 'www.freebase.com/book/written_work/subjects',
          'object': 'www.freebase.com/m/01cj3p',
          'subject': 'www.freebase.com/m/04whkz5'
        }, {
          'question': 'to what release does the release track cardiac arrest come from',
          'relation': 'www.freebase.com/music/release_track/release',
          'object': 'www.freebase.com/m/0sjc7c1',
          'subject': 'www.freebase.com/m/0tp2p24'
        }]
    """
    download_file_maybe_extract(url=url, directory=directory, check_files=check_files)

    ret = []
    splits = [(train, train_filename), (dev, dev_filename), (test, test_filename)]
    splits = [f for (requested, f) in splits if requested]
    for filename in splits:
        full_path = os.path.join(directory, extracted_name, filename)
        data = pd.read_csv(
            full_path, header=None, sep='\t', names=['subject', 'relation', 'object', 'question'])
        ret.append(
            Dataset([{
                'question': row['question'],
                'relation': row['relation'],
                'object': row['object'],
                'subject': row['subject'],
            } for _, row in data.iterrows()]))

    if len(ret) == 1:
        return ret[0]
    else:
        return tuple(ret)


Instruction:
Can you generate the documentation for the following Python 3 function
def forward(self, x):
        if not self.training or not self.p:
            return x
        x = x.clone()
        mask = x.new_empty(1, x.size(1), x.size(2), requires_grad=False).bernoulli_(1 - self.p)
        mask = mask.div_(1 - self.p)
        mask = mask.expand_as(x)
        return x * mask

Response: Forward computation of the n - grams.


Instruction:
Can you tell what is the following Python 3 function doing
def pad_tensor(tensor, length, padding_index=DEFAULT_PADDING_INDEX):
    n_padding = length - tensor.shape[0]
    assert n_padding >= 0
    if n_padding == 0:
        return tensor
    padding = tensor.new(n_padding, *tensor.shape[1:]).fill_(padding_index)
    return torch.cat((tensor, padding), dim=0)

Response: Pad a tensor up to length with padding_index.


Instruction:
Write a Python 3 function that can
pad a list of tensors with padding_index.

Response:
def stack_and_pad_tensors(batch, padding_index=DEFAULT_PADDING_INDEX, dim=0):
    """ Pad a :class:`list` of ``tensors`` (``batch``) with ``padding_index``.

    Args:
        batch (:class:`list` of :class:`torch.Tensor`): Batch of tensors to pad.
        padding_index (int, optional): Index to pad tensors with.
        dim (int, optional): Dimension on to which to concatenate the batch of tensors.

    Returns
        torch.Tensor, torch.Tensor: Padded tensors and original lengths of tensors.
    """
    lengths = [tensor.shape[0] for tensor in batch]
    max_len = max(lengths)
    padded = [pad_tensor(tensor, max_len, padding_index) for tensor in batch]
    lengths = torch.tensor(lengths)
    padded = torch.stack(padded, dim=dim).contiguous()
    for _ in range(dim):
        lengths = lengths.unsqueeze(0)
    return padded, lengths


Instruction:
Write a Python 3 function that can
return a batch of text encoded into a single tensor.

Response:
def batch_encode(self, iterator, *args, dim=0, **kwargs):
        """
        Args:
            iterator (iterator): Batch of text to encode.
            *args: Arguments passed onto ``Encoder.__init__``.
            dim (int, optional): Dimension along which to concatenate tensors.
            **kwargs: Keyword arguments passed onto ``Encoder.__init__``.

        Returns
            torch.Tensor, list of int: Encoded and padded batch of sequences; Original lengths of
                sequences.
        """
        return stack_and_pad_tensors(
            super().batch_encode(iterator), padding_index=self.padding_index, dim=dim)


Instruction:
How would you implement a function in Python 3 that
decodes a batch of encoded sequences.

Response:
def batch_decode(self, tensor, lengths, dim=0, *args, **kwargs):
        """
        Args:
            batch (list of :class:`torch.Tensor`): Batch of encoded sequences.
            lengths (list of int): Original lengths of sequences.
            dim (int, optional): Dimension along which to split tensors.
            *args: Arguments passed to ``decode``.
            **kwargs: Key word arguments passed to ``decode``.

        Returns:
            list: Batch of decoded sequences.
        """
        return super().batch_decode(
            [t.squeeze(0)[:l] for t, l in zip(tensor.split(1, dim=dim), lengths)])


Instruction:
How would you code a function in Python 3 to
encode a sequence into a new version of the sequence.

Response:
def encode(self, sequence):
        """ Encodes a ``sequence``.

        Args:
            sequence (str): String ``sequence`` to encode.

        Returns:
            torch.Tensor: Encoding of the ``sequence``.
        """
        sequence = super().encode(sequence)
        sequence = self.tokenize(sequence)
        vector = [self.stoi.get(token, self.unknown_index) for token in sequence]
        if self.append_eos:
            vector.append(self.eos_index)
        return torch.tensor(vector)


Instruction:
Here you have a function in Python 3, explain what it does
def decode(self, encoded):
        encoded = super().decode(encoded)
        tokens = [self.itos[index] for index in encoded]
        return self.detokenize(tokens)

Response: Decodes a tensor into a sequence.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _reporthook(t):
    last_b = [0]

    def inner(b=1, bsize=1, tsize=None):
        """
        Args:
            b (int, optional): Number of blocks just transferred [default: 1].
            bsize (int, optional): Size of each block (in tqdm units) [default: 1].
            tsize (int, optional): Total size (in tqdm units). If [default: None] remains unchanged.
        """
        if tsize is not None:
            t.total = tsize
        t.update((b - last_b[0]) * bsize)
        last_b[0] = b

    return inner

Response: A function that returns a function that returns a tuple of tuples with the first element of the tuple as first argument and the second element as second argument.


Instruction:
Can you write a function in Python 3 where it
downloads a file from Google drive.

Response:
def _download_file_from_drive(filename, url):  # pragma: no cover
    """ Download filename from google drive unless it's already in directory.

    Args:
        filename (str): Name of the file to download to (do nothing if it already exists).
        url (str): URL to download from.
    """
    confirm_token = None

    # Since the file is big, drive will scan it for virus and take it to a
    # warning page. We find the confirm token on this page and append it to the
    # URL to start the download process.
    confirm_token = None
    session = requests.Session()
    response = session.get(url, stream=True)
    for k, v in response.cookies.items():
        if k.startswith("download_warning"):
            confirm_token = v

    if confirm_token:
        url = url + "&confirm=" + confirm_token

    logger.info("Downloading %s to %s" % (url, filename))

    response = session.get(url, stream=True)
    # Now begin the download.
    chunk_size = 16 * 1024
    with open(filename, "wb") as f:
        for chunk in response.iter_content(chunk_size):
            if chunk:
                f.write(chunk)

    # Print newline to clear the carriage return from the download progress
    statinfo = os.stat(filename)
    logger.info("Successfully downloaded %s, %s bytes." % (filename, statinfo.st_size))


Instruction:
Implement a Python 3 function for
extracting a compressed file to a directory.

Response:
def _maybe_extract(compressed_filename, directory, extension=None):
    """ Extract a compressed file to ``directory``.

    Args:
        compressed_filename (str): Compressed file.
        directory (str): Extract to directory.
        extension (str, optional): Extension of the file; Otherwise, attempts to extract extension
            from the filename.
    """
    logger.info('Extracting {}'.format(compressed_filename))

    if extension is None:
        basename = os.path.basename(compressed_filename)
        extension = basename.split('.', 1)[1]

    if 'zip' in extension:
        with zipfile.ZipFile(compressed_filename, "r") as zip_:
            zip_.extractall(directory)
    elif 'tar' in extension or 'tgz' in extension:
        with tarfile.open(compressed_filename, mode='r') as tar:
            tar.extractall(path=directory)

    logger.info('Extracted {}'.format(compressed_filename))


Instruction:
Make a summary of the following Python 3 code
def _get_filename_from_url(url):
    parse = urlparse(url)
    return os.path.basename(parse.path)

Response: Return a filename from a URL


Instruction:
Write a Python 3 function for
downloading the file at url to directory if tar or zip.

Response:
def download_file_maybe_extract(url, directory, filename=None, extension=None, check_files=[]):
    """ Download the file at ``url`` to ``directory``. Extract to ``directory`` if tar or zip.

    Args:
        url (str): Url of file.
        directory (str): Directory to download to.
        filename (str, optional): Name of the file to download; Otherwise, a filename is extracted
            from the url.
        extension (str, optional): Extension of the file; Otherwise, attempts to extract extension
            from the filename.
        check_files (list of str): Check if these files exist, ensuring the download succeeded.
            If these files exist before the download, the download is skipped.

    Returns:
        (str): Filename of download file.

    Raises:
        ValueError: Error if one of the ``check_files`` are not found following the download.
    """
    if filename is None:
        filename = _get_filename_from_url(url)

    filepath = os.path.join(directory, filename)
    check_files = [os.path.join(directory, f) for f in check_files]

    if len(check_files) > 0 and _check_download(*check_files):
        return filepath

    if not os.path.isdir(directory):
        os.makedirs(directory)

    logger.info('Downloading {}'.format(filename))

    # Download
    if 'drive.google.com' in url:
        _download_file_from_drive(filepath, url)
    else:
        with tqdm(unit='B', unit_scale=True, miniters=1, desc=filename) as t:
            urllib.request.urlretrieve(url, filename=filepath, reporthook=_reporthook(t))

    _maybe_extract(compressed_filename=filepath, directory=directory, extension=extension)

    if not _check_download(*check_files):
        raise ValueError('[DOWNLOAD FAILED] `*check_files` not found')

    return filepath


Instruction:
Can you tell what is the following Python 3 function doing
def download_files_maybe_extract(urls, directory, check_files=[]):
    check_files = [os.path.join(directory, f) for f in check_files]
    if _check_download(*check_files):
        return

    for url in urls:
        download_file_maybe_extract(url=url, directory=directory)

    if not _check_download(*check_files):
        raise ValueError('[DOWNLOAD FAILED] `*check_files` not found')

Response: Download the files at urls to directory if tar or zip.


Instruction:
Write a Python 3 function that can
get the accuracy between two tensors.

Response:
def get_accuracy(targets, outputs, k=1, ignore_index=None):
    """ Get the accuracy top-k accuracy between two tensors.

    Args:
      targets (1 - 2D :class:`torch.Tensor`): Target or true vector against which to measure
          saccuracy
      outputs (1 - 3D :class:`torch.Tensor`): Prediction or output vector
      ignore_index (int, optional): Specifies a target index that is ignored

    Returns:
      :class:`tuple` consisting of accuracy (:class:`float`), number correct (:class:`int`) and
      total (:class:`int`)

    Example:

        >>> import torch
        >>> from torchnlp.metrics import get_accuracy
        >>> targets = torch.LongTensor([1, 2, 3, 4, 5])
        >>> outputs = torch.LongTensor([1, 2, 2, 3, 5])
        >>> accuracy, n_correct, n_total = get_accuracy(targets, outputs, ignore_index=3)
        >>> accuracy
        0.8
        >>> n_correct
        4
        >>> n_total
        5
    """
    n_correct = 0.0
    for target, output in zip(targets, outputs):
        if not torch.is_tensor(target) or is_scalar(target):
            target = torch.LongTensor([target])

        if not torch.is_tensor(output) or is_scalar(output):
            output = torch.LongTensor([[output]])

        predictions = output.topk(k=min(k, len(output)), dim=0)[0]
        for prediction in predictions:
            if torch_equals_ignore_index(
                    target.squeeze(), prediction.squeeze(), ignore_index=ignore_index):
                n_correct += 1
                break

    return n_correct / len(targets), int(n_correct), len(targets)


Instruction:
Here you have a function in Python 3, explain what it does
def get_token_accuracy(targets, outputs, ignore_index=None):
    n_correct = 0.0
    n_total = 0.0
    for target, output in zip(targets, outputs):
        if not torch.is_tensor(target) or is_scalar(target):
            target = torch.LongTensor([target])

        if not torch.is_tensor(output) or is_scalar(output):
            output = torch.LongTensor([[output]])

        if len(target.size()) != len(output.size()):
            prediction = output.max(dim=0)[0].view(-1)
        else:
            prediction = output

        if ignore_index is not None:
            mask = target.ne(ignore_index)
            n_correct += prediction.eq(target).masked_select(mask).sum().item()
            n_total += mask.sum().item()
        else:
            n_total += len(target)
            n_correct += prediction.eq(target).sum().item()

    return n_correct / n_total, n_correct, n_total

Response: Get the accuracy token accuracy between two tensors.


Instruction:
Given the following Python 3 function, write the documentation
def get_tensors(object_):
    if torch.is_tensor(object_):
        return [object_]
    elif isinstance(object_, (str, float, int)):
        return []

    tensors = set()

    if isinstance(object_, collections.abc.Mapping):
        for value in object_.values():
            tensors.update(get_tensors(value))
    elif isinstance(object_, collections.abc.Iterable):
        for value in object_:
            tensors.update(get_tensors(value))
    else:
        members = [
            value for key, value in inspect.getmembers(object_)
            if not isinstance(value, (collections.abc.Callable, type(None)))
        ]
        tensors.update(get_tensors(members))

    return tensors

Response: Returns a list of tensors that are associated with object_.


Instruction:
Make a summary of the following Python 3 code
def sampler_to_iterator(dataset, sampler):
    for sample in sampler:
        if isinstance(sample, (list, tuple)):
            # yield a batch
            yield [dataset[i] for i in sample]
        else:
            # yield a single example
            yield dataset[sample]

Response:
Given a batch sampler or sampler returns examples instead of indices
AttributeNames


Instruction:
Make a summary of the following Python 3 code
def resplit_datasets(dataset, other_dataset, random_seed=None, split=None):
    # Prevent circular dependency
    from torchnlp.datasets import Dataset

    concat = dataset.rows + other_dataset.rows
    shuffle(concat, random_seed=random_seed)
    if split is None:
        return Dataset(concat[:len(dataset)]), Dataset(concat[len(dataset):])
    else:
        split = max(min(round(len(concat) * split), len(concat)), 0)
        return Dataset(concat[:split]), Dataset(concat[split:])

Response: Deterministic shuffle and split algorithm.


Instruction:
How would you explain what the following Python 3 function does
def torch_equals_ignore_index(tensor, tensor_other, ignore_index=None):
    if ignore_index is not None:
        assert tensor.size() == tensor_other.size()
        mask_arr = tensor.ne(ignore_index)
        tensor = tensor.masked_select(mask_arr)
        tensor_other = tensor_other.masked_select(mask_arr)

    return torch.equal(tensor, tensor_other)

Response: Compute torch. equal with the optional mask parameter.


Instruction:
Can you create a Python 3 function that
gives a list of lengths create a batch mask.

Response:
def lengths_to_mask(*lengths, **kwargs):
    """ Given a list of lengths, create a batch mask.

    Example:
        >>> lengths_to_mask([1, 2, 3])
        tensor([[1, 0, 0],
                [1, 1, 0],
                [1, 1, 1]], dtype=torch.uint8)
        >>> lengths_to_mask([1, 2, 2], [1, 2, 2])
        tensor([[[1, 0],
                 [0, 0]],
        <BLANKLINE>
                [[1, 1],
                 [1, 1]],
        <BLANKLINE>
                [[1, 1],
                 [1, 1]]], dtype=torch.uint8)

    Args:
        *lengths (list of int or torch.Tensor)
        **kwargs: Keyword arguments passed to ``torch.zeros`` upon initially creating the returned
          tensor.

    Returns:
        torch.ByteTensor
    """
    # Squeeze to deal with random additional dimensions
    lengths = [l.squeeze().tolist() if torch.is_tensor(l) else l for l in lengths]

    # For cases where length is a scalar, this needs to convert it to a list.
    lengths = [l if isinstance(l, list) else [l] for l in lengths]
    assert all(len(l) == len(lengths[0]) for l in lengths)
    batch_size = len(lengths[0])
    other_dimensions = tuple([int(max(l)) for l in lengths])
    mask = torch.zeros(batch_size, *other_dimensions, **kwargs)
    for i, length in enumerate(zip(*tuple(lengths))):
        mask[i][[slice(int(l)) for l in length]].fill_(1)
    return mask.byte()


Instruction:
How would you implement a function in Python 3 that
collates a list of types k with tensors.

Response:
def collate_tensors(batch, stack_tensors=torch.stack):
    """ Collate a list of type ``k`` (dict, namedtuple, list, etc.) with tensors.

    Inspired by:
    https://github.com/pytorch/pytorch/blob/master/torch/utils/data/_utils/collate.py#L31

    Args:
        batch (list of k): List of rows of type ``k``.
        stack_tensors (callable): Function to stack tensors into a batch.

    Returns:
        k: Collated batch of type ``k``.

    Example use case:
        This is useful with ``torch.utils.data.dataloader.DataLoader`` which requires a collate
        function. Typically, when collating sequences you'd set
        ``collate_fn=partial(collate_tensors, stack_tensors=encoders.text.stack_and_pad_tensors)``.

    Example:

        >>> import torch
        >>> batch = [
        ...   { 'column_a': torch.randn(5), 'column_b': torch.randn(5) },
        ...   { 'column_a': torch.randn(5), 'column_b': torch.randn(5) },
        ... ]
        >>> collated = collate_tensors(batch)
        >>> {k: t.size() for (k, t) in collated.items()}
        {'column_a': torch.Size([2, 5]), 'column_b': torch.Size([2, 5])}
    """
    if all([torch.is_tensor(b) for b in batch]):
        return stack_tensors(batch)
    if (all([isinstance(b, dict) for b in batch]) and
            all([b.keys() == batch[0].keys() for b in batch])):
        return {key: collate_tensors([d[key] for d in batch], stack_tensors) for key in batch[0]}
    elif all([is_namedtuple(b) for b in batch]):  # Handle ``namedtuple``
        return batch[0].__class__(**collate_tensors([b._asdict() for b in batch], stack_tensors))
    elif all([isinstance(b, list) for b in batch]):
        # Handle list of lists such each list has some column to be batched, similar to:
        # [['a', 'b'], ['a', 'b']] → [['a', 'a'], ['b', 'b']]
        transposed = zip(*batch)
        return [collate_tensors(samples, stack_tensors) for samples in transposed]
    else:
        return batch


Instruction:
Create a Python 3 function to
apply torch. Tensor. to a list of tensors and returns a new list of tuples.

Response:
def tensors_to(tensors, *args, **kwargs):
    """ Apply ``torch.Tensor.to`` to tensors in a generic data structure.

    Inspired by:
    https://github.com/pytorch/pytorch/blob/master/torch/utils/data/_utils/collate.py#L31

    Args:
        tensors (tensor, dict, list, namedtuple or tuple): Data structure with tensor values to
            move.
        *args: Arguments passed to ``torch.Tensor.to``.
        **kwargs: Keyword arguments passed to ``torch.Tensor.to``.

    Example use case:
        This is useful as a complementary function to ``collate_tensors``. Following collating,
        it's important to move your tensors to the appropriate device.

    Returns:
        The inputted ``tensors`` with ``torch.Tensor.to`` applied.

    Example:

        >>> import torch
        >>> batch = [
        ...   { 'column_a': torch.randn(5), 'column_b': torch.randn(5) },
        ...   { 'column_a': torch.randn(5), 'column_b': torch.randn(5) },
        ... ]
        >>> tensors_to(batch, torch.device('cpu'))  # doctest: +ELLIPSIS
        [{'column_a': tensor(...}]
    """
    if torch.is_tensor(tensors):
        return tensors.to(*args, **kwargs)
    elif isinstance(tensors, dict):
        return {k: tensors_to(v, *args, **kwargs) for k, v in tensors.items()}
    elif hasattr(tensors, '_asdict') and isinstance(tensors, tuple):  # Handle ``namedtuple``
        return tensors.__class__(**tensors_to(tensors._asdict(), *args, **kwargs))
    elif isinstance(tensors, list):
        return [tensors_to(t, *args, **kwargs) for t in tensors]
    elif isinstance(tensors, tuple):
        return tuple([tensors_to(t, *args, **kwargs) for t in tensors])
    else:
        return tensors


Instruction:
How would you code a function in Python 3 to
load the Zero dataset.

Response:
def zero_dataset(train=False, dev=False, test=False, train_rows=256, dev_rows=64, test_rows=64):
    """
    Load the Zero dataset.

    The Zero dataset is a simple task of predicting zero from zero. This dataset is useful for
    integration testing. The extreme simplicity of the dataset allows for models to learn the task
    quickly allowing for quick end-to-end testing.

    Args:
        train (bool, optional): If to load the training split of the dataset.
        dev (bool, optional): If to load the development split of the dataset.
        test (bool, optional): If to load the test split of the dataset.
        train_rows (int, optional): Number of training rows to generate.
        dev_rows (int, optional): Number of development rows to generate.
        test_rows (int, optional): Number of test rows to generate.

    Returns:
        :class:`tuple` of :class:`torchnlp.datasets.Dataset` or :class:`torchnlp.datasets.Dataset`:
        Returns between one and all dataset splits (train, dev and test) depending on if their
        respective boolean argument is ``True``.

    Example:
        >>> from torchnlp.datasets import zero_dataset
        >>> train = zero_dataset(train=True)
        >>> train[0:2]
        [{'source': '0', 'target': '0'}, {'source': '0', 'target': '0'}]
    """
    ret = []
    for is_requested, n_rows in [(train, train_rows), (dev, dev_rows), (test, test_rows)]:
        if not is_requested:
            continue
        rows = [{'source': str(0), 'target': str(0)} for i in range(n_rows)]
        ret.append(Dataset(rows))

    if len(ret) == 1:
        return ret[0]
    else:
        return tuple(ret)


Instruction:
Write a Python 3 script to
decode a tensor into a sequence.

Response:
def decode(self, encoded):
        """ Decodes a tensor into a sequence.

        Args:
            encoded (torch.Tensor): Encoded sequence.

        Returns:
            str: Sequence decoded from ``encoded``.
        """
        encoded = super().decode(encoded)
        return self.tokenizer.decode([self.itos[index] for index in encoded])


Instruction:
Make a summary of the following Python 3 code
def encode(self, label):
        label = super().encode(label)

        return torch.tensor(self.stoi.get(label, self.unknown_index))

Response: Encodes a label into a new version of the label.


Instruction:
Here you have a function in Python 3, explain what it does
def batch_encode(self, iterator, *args, dim=0, **kwargs):
        return torch.stack(super().batch_encode(iterator, *args, **kwargs), dim=dim)

Response: Returns a batch of encoded labels.


Instruction:
Given the following Python 3 function, write the documentation
def decode(self, encoded):
        encoded = super().decode(encoded)

        if encoded.numel() > 1:
            raise ValueError(
                '``decode`` decodes one label at a time, use ``batch_decode`` instead.')

        return self.itos[encoded.squeeze().item()]

Response: Decodes an encoded label.


Instruction:
Create a Python 3 function to
batch decoding of a single object.

Response:
def batch_decode(self, tensor, *args, dim=0, **kwargs):
        """
        Args:
            tensor (torch.Tensor): Batch of tensors.
            *args: Arguments passed to ``Encoder.batch_decode``.
            dim (int, optional): Dimension along which to split tensors.
            **kwargs: Keyword arguments passed to ``Encoder.batch_decode``.

        Returns:
            list: Batch of decoded labels.
        """
        return super().batch_decode([t.squeeze(0) for t in tensor.split(1, dim=dim)])


Instruction:
Write a Python 3 function for
loading the Count dataset.

Response:
def count_dataset(train=False,
                  dev=False,
                  test=False,
                  train_rows=10000,
                  dev_rows=1000,
                  test_rows=1000,
                  seq_max_length=10):
    """
    Load the Count dataset.

    The Count dataset is a simple task of counting the number of integers in a sequence. This
    dataset is useful for testing implementations of sequence to label models.

    Args:
        train (bool, optional): If to load the training split of the dataset.
        dev (bool, optional): If to load the development split of the dataset.
        test (bool, optional): If to load the test split of the dataset.
        train_rows (int, optional): Number of training rows to generate.
        dev_rows (int, optional): Number of development rows to generate.
        test_rows (int, optional): Number of test rows to generate.
        seq_max_length (int, optional): Maximum sequence length.

    Returns:
        :class:`tuple` of :class:`torchnlp.datasets.Dataset` or :class:`torchnlp.datasets.Dataset`:
        Returns between one and all dataset splits (train, dev and test) depending on if their
        respective boolean argument is ``True``.

    Example:
        >>> import random
        >>> random.seed(321)
        >>>
        >>> from torchnlp.datasets import count_dataset
        >>> train = count_dataset(train=True)
        >>> train[0:2]
        [{'numbers': '6 2 5 8 7', 'count': '5'}, {'numbers': '3 9 7 6 6 7', 'count': '6'}]
    """
    ret = []
    for is_requested, n_rows in [(train, train_rows), (dev, dev_rows), (test, test_rows)]:
        rows = []
        for i in range(n_rows):
            length = random.randint(1, seq_max_length)
            seq = []
            for _ in range(length):
                seq.append(str(random.randint(0, 9)))
            input_ = ' '.join(seq)
            rows.append({'numbers': input_, 'count': str(length)})

        # NOTE: Given that `random.randint` is deterministic with the same `random_seed` we need
        # to allow the random generator to create the train, dev and test dataset in order.
        # Otherwise, `reverse(train=True)` and `reverse(test=True)` would share the first 1000 rows.
        if not is_requested:
            continue

        ret.append(Dataset(rows))

    if len(ret) == 1:
        return ret[0]
    else:
        return tuple(ret)


Instruction:
Explain what the following Python 3 code does
def multi30k_dataset(directory='data/multi30k/',
                     train=False,
                     dev=False,
                     test=False,
                     train_filename='train',
                     dev_filename='val',
                     test_filename='test',
                     check_files=['train.de', 'val.de'],
                     urls=[
                         'http://www.quest.dcs.shef.ac.uk/wmt16_files_mmt/training.tar.gz',
                         'http://www.quest.dcs.shef.ac.uk/wmt16_files_mmt/validation.tar.gz',
                         'http://www.quest.dcs.shef.ac.uk/wmt16_files_mmt/mmt16_task1_test.tar.gz'
                     ]):
    """
    Load the WMT 2016 machine translation dataset.

    As a translation task, this task consists in translating English sentences that describe an
    image into German, given the English sentence itself. As training and development data, we
    provide 29,000 and 1,014 triples respectively, each containing an English source sentence, its
    German human translation. As test data, we provide a new set of 1,000 tuples containing an
    English description.

    Status:
        Host ``www.quest.dcs.shef.ac.uk`` forgot to update their SSL
        certificate; therefore, this dataset does not download securely.

    References:
        * http://www.statmt.org/wmt16/multimodal-task.html
        * http://shannon.cs.illinois.edu/DenotationGraph/

    **Citation**
    ::

        @article{elliott-EtAl:2016:VL16,
            author    = {{Elliott}, D. and {Frank}, S. and {Sima'an}, K. and {Specia}, L.},
            title     = {Multi30K: Multilingual English-German Image Descriptions},
            booktitle = {Proceedings of the 5th Workshop on Vision and Language},
            year      = {2016},
            pages     = {70--74},
            year      = 2016
        }

    Args:
        directory (str, optional): Directory to cache the dataset.
        train (bool, optional): If to load the training split of the dataset.
        dev (bool, optional): If to load the dev split of the dataset.
        test (bool, optional): If to load the test split of the dataset.
        train_directory (str, optional): The directory of the training split.
        dev_directory (str, optional): The directory of the dev split.
        test_directory (str, optional): The directory of the test split.
        check_files (str, optional): Check if these files exist, then this download was successful.
        urls (str, optional): URLs to download.

    Returns:
        :class:`tuple` of :class:`torchnlp.datasets.Dataset` or :class:`torchnlp.datasets.Dataset`:
        Returns between one and all dataset splits (train, dev and test) depending on if their
        respective boolean argument is ``True``.

    Example:
        >>> from torchnlp.datasets import multi30k_dataset  # doctest: +SKIP
        >>> train = multi30k_dataset(train=True)  # doctest: +SKIP
        >>> train[:2]  # doctest: +SKIP
        [{
          'en': 'Two young, White males are outside near many bushes.',
          'de': 'Zwei junge weiße Männer sind im Freien in der Nähe vieler Büsche.'
        }, {
          'en': 'Several men in hard hatsare operating a giant pulley system.',
          'de': 'Mehrere Männer mit Schutzhelmen bedienen ein Antriebsradsystem.'
        }]
    """
    download_files_maybe_extract(urls=urls, directory=directory, check_files=check_files)

    ret = []
    splits = [(train, train_filename), (dev, dev_filename), (test, test_filename)]
    splits = [f for (requested, f) in splits if requested]

    for filename in splits:
        examples = []

        en_path = os.path.join(directory, filename + '.en')
        de_path = os.path.join(directory, filename + '.de')
        en_file = [l.strip() for l in open(en_path, 'r', encoding='utf-8')]
        de_file = [l.strip() for l in open(de_path, 'r', encoding='utf-8')]
        assert len(en_file) == len(de_file)
        for i in range(len(en_file)):
            if en_file[i] != '' and de_file[i] != '':
                examples.append({'en': en_file[i], 'de': de_file[i]})

        ret.append(Dataset(examples))

    if len(ret) == 1:
        return ret[0]
    else:
        return tuple(ret)

Response: Load the multi30k dataset.