unicore/data/__init__.py

# Copyright (c) DP Technology.
# Copyright (c) Facebook, Inc. and its affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.
"""isort:skip_file"""


from .unicore_dataset import UnicoreDataset

from .base_wrapper_dataset import BaseWrapperDataset

from .append_token_dataset import AppendTokenDataset
from .dictionary import Dictionary
from .lru_cache_dataset import LRUCacheDataset
from .mask_tokens_dataset import MaskTokensDataset
from .bert_tokenize_dataset import BertTokenizeDataset
from .tokenize_dataset import TokenizeDataset
from .nested_dictionary_dataset import NestedDictionaryDataset
from .numel_dataset import NumelDataset
from .num_samples_dataset import NumSamplesDataset
from .pad_dataset import LeftPadDataset, PadDataset, RightPadDataset, RightPadDataset2D
from .prepend_token_dataset import PrependTokenDataset
from .raw_dataset import RawLabelDataset, RawArrayDataset, RawNumpyDataset
from .lmdb_dataset import LMDBDataset
from .sort_dataset import SortDataset, EpochShuffleDataset
from .from_numpy_dataset import FromNumpyDataset

from .iterators import (
    CountingIterator,
    EpochBatchIterator,
    GroupedIterator,
    ShardedIterator,
)

__all__ = []