Instruction:
Make a summary of the following Python 3 code
def _load_mnist_dataset(shape, path, name='mnist', url='http://yann.lecun.com/exdb/mnist/'):
    path = os.path.join(path, name)

    # Define functions for loading mnist-like data's images and labels.
    # For convenience, they also download the requested files if needed.
    def load_mnist_images(path, filename):
        filepath = maybe_download_and_extract(filename, path, url)

        logging.info(filepath)
        # Read the inputs in Yann LeCun's binary format.
        with gzip.open(filepath, 'rb') as f:
            data = np.frombuffer(f.read(), np.uint8, offset=16)
        # The inputs are vectors now, we reshape them to monochrome 2D images,
        # following the shape convention: (examples, channels, rows, columns)
        data = data.reshape(shape)
        # The inputs come as bytes, we convert them to float32 in range [0,1].
        # (Actually to range [0, 255/256], for compatibility to the version
        # provided at http://deeplearning.net/data/mnist/mnist.pkl.gz.)
        return data / np.float32(256)

    def load_mnist_labels(path, filename):
        filepath = maybe_download_and_extract(filename, path, url)
        # Read the labels in Yann LeCun's binary format.
        with gzip.open(filepath, 'rb') as f:
            data = np.frombuffer(f.read(), np.uint8, offset=8)
        # The labels are vectors of integers now, that's exactly what we want.
        return data

    # Download and read the training and test set images and labels.
    logging.info("Load or Download {0} > {1}".format(name.upper(), path))
    X_train = load_mnist_images(path, 'train-images-idx3-ubyte.gz')
    y_train = load_mnist_labels(path, 'train-labels-idx1-ubyte.gz')
    X_test = load_mnist_images(path, 't10k-images-idx3-ubyte.gz')
    y_test = load_mnist_labels(path, 't10k-labels-idx1-ubyte.gz')

    # We reserve the last 10000 training examples for validation.
    X_train, X_val = X_train[:-10000], X_train[-10000:]
    y_train, y_val = y_train[:-10000], y_train[-10000:]

    # We just return all the arrays in order, as expected in main().
    # (It doesn't matter how we do this as long as we can read them again.)
    X_train = np.asarray(X_train, dtype=np.float32)
    y_train = np.asarray(y_train, dtype=np.int32)
    X_val = np.asarray(X_val, dtype=np.float32)
    y_val = np.asarray(y_val, dtype=np.int32)
    X_test = np.asarray(X_test, dtype=np.float32)
    y_test = np.asarray(y_test, dtype=np.int32)
    return X_train, y_train, X_val, y_val, X_test, y_test

Response: Load mnist - like dataset.


Instruction:
Create a Python 3 function to
load CIFAR - 10 dataset.

Response:
def load_cifar10_dataset(shape=(-1, 32, 32, 3), path='data', plotable=False):
    """Load CIFAR-10 dataset.

    It consists of 60000 32x32 colour images in 10 classes, with
    6000 images per class. There are 50000 training images and 10000 test images.

    The dataset is divided into five training batches and one test batch, each with
    10000 images. The test batch contains exactly 1000 randomly-selected images from
    each class. The training batches contain the remaining images in random order,
    but some training batches may contain more images from one class than another.
    Between them, the training batches contain exactly 5000 images from each class.

    Parameters
    ----------
    shape : tupe
        The shape of digit images e.g. (-1, 3, 32, 32) and (-1, 32, 32, 3).
    path : str
        The path that the data is downloaded to, defaults is ``data/cifar10/``.
    plotable : boolean
        Whether to plot some image examples, False as default.

    Examples
    --------
    >>> X_train, y_train, X_test, y_test = tl.files.load_cifar10_dataset(shape=(-1, 32, 32, 3))

    References
    ----------
    - `CIFAR website <https://www.cs.toronto.edu/~kriz/cifar.html>`__
    - `Data download link <https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz>`__
    - `<https://teratail.com/questions/28932>`__

    """
    path = os.path.join(path, 'cifar10')
    logging.info("Load or Download cifar10 > {}".format(path))

    # Helper function to unpickle the data
    def unpickle(file):
        fp = open(file, 'rb')
        if sys.version_info.major == 2:
            data = pickle.load(fp)
        elif sys.version_info.major == 3:
            data = pickle.load(fp, encoding='latin-1')
        fp.close()
        return data

    filename = 'cifar-10-python.tar.gz'
    url = 'https://www.cs.toronto.edu/~kriz/'
    # Download and uncompress file
    maybe_download_and_extract(filename, path, url, extract=True)

    # Unpickle file and fill in data
    X_train = None
    y_train = []
    for i in range(1, 6):
        data_dic = unpickle(os.path.join(path, 'cifar-10-batches-py/', "data_batch_{}".format(i)))
        if i == 1:
            X_train = data_dic['data']
        else:
            X_train = np.vstack((X_train, data_dic['data']))
        y_train += data_dic['labels']

    test_data_dic = unpickle(os.path.join(path, 'cifar-10-batches-py/', "test_batch"))
    X_test = test_data_dic['data']
    y_test = np.array(test_data_dic['labels'])

    if shape == (-1, 3, 32, 32):
        X_test = X_test.reshape(shape)
        X_train = X_train.reshape(shape)
    elif shape == (-1, 32, 32, 3):
        X_test = X_test.reshape(shape, order='F')
        X_train = X_train.reshape(shape, order='F')
        X_test = np.transpose(X_test, (0, 2, 1, 3))
        X_train = np.transpose(X_train, (0, 2, 1, 3))
    else:
        X_test = X_test.reshape(shape)
        X_train = X_train.reshape(shape)

    y_train = np.array(y_train)

    if plotable:
        logging.info('\nCIFAR-10')
        fig = plt.figure(1)

        logging.info('Shape of a training image: X_train[0] %s' % X_train[0].shape)

        plt.ion()  # interactive mode
        count = 1
        for _ in range(10):  # each row
            for _ in range(10):  # each column
                _ = fig.add_subplot(10, 10, count)
                if shape == (-1, 3, 32, 32):
                    # plt.imshow(X_train[count-1], interpolation='nearest')
                    plt.imshow(np.transpose(X_train[count - 1], (1, 2, 0)), interpolation='nearest')
                    # plt.imshow(np.transpose(X_train[count-1], (2, 1, 0)), interpolation='nearest')
                elif shape == (-1, 32, 32, 3):
                    plt.imshow(X_train[count - 1], interpolation='nearest')
                    # plt.imshow(np.transpose(X_train[count-1], (1, 0, 2)), interpolation='nearest')
                else:
                    raise Exception("Do not support the given 'shape' to plot the image examples")
                plt.gca().xaxis.set_major_locator(plt.NullLocator())  # 不显示刻度(tick)
                plt.gca().yaxis.set_major_locator(plt.NullLocator())
                count = count + 1
        plt.draw()  # interactive mode
        plt.pause(3)  # interactive mode

        logging.info("X_train: %s" % X_train.shape)
        logging.info("y_train: %s" % y_train.shape)
        logging.info("X_test:  %s" % X_test.shape)
        logging.info("y_test:  %s" % y_test.shape)

    X_train = np.asarray(X_train, dtype=np.float32)
    X_test = np.asarray(X_test, dtype=np.float32)
    y_train = np.asarray(y_train, dtype=np.int32)
    y_test = np.asarray(y_test, dtype=np.int32)

    return X_train, y_train, X_test, y_test


Instruction:
Given the following Python 3 function, write the documentation
def load_cropped_svhn(path='data', include_extra=True):
    start_time = time.time()

    path = os.path.join(path, 'cropped_svhn')
    logging.info("Load or Download Cropped SVHN > {} | include extra images: {}".format(path, include_extra))
    url = "http://ufldl.stanford.edu/housenumbers/"

    np_file = os.path.join(path, "train_32x32.npz")
    if file_exists(np_file) is False:
        filename = "train_32x32.mat"
        filepath = maybe_download_and_extract(filename, path, url)
        mat = sio.loadmat(filepath)
        X_train = mat['X'] / 255.0  # to [0, 1]
        X_train = np.transpose(X_train, (3, 0, 1, 2))
        y_train = np.squeeze(mat['y'], axis=1)
        y_train[y_train == 10] = 0  # replace 10 to 0
        np.savez(np_file, X=X_train, y=y_train)
        del_file(filepath)
    else:
        v = np.load(np_file)
        X_train = v['X']
        y_train = v['y']
    logging.info("  n_train: {}".format(len(y_train)))

    np_file = os.path.join(path, "test_32x32.npz")
    if file_exists(np_file) is False:
        filename = "test_32x32.mat"
        filepath = maybe_download_and_extract(filename, path, url)
        mat = sio.loadmat(filepath)
        X_test = mat['X'] / 255.0
        X_test = np.transpose(X_test, (3, 0, 1, 2))
        y_test = np.squeeze(mat['y'], axis=1)
        y_test[y_test == 10] = 0
        np.savez(np_file, X=X_test, y=y_test)
        del_file(filepath)
    else:
        v = np.load(np_file)
        X_test = v['X']
        y_test = v['y']
    logging.info("  n_test: {}".format(len(y_test)))

    if include_extra:
        logging.info("  getting extra 531131 images, please wait ...")
        np_file = os.path.join(path, "extra_32x32.npz")
        if file_exists(np_file) is False:
            logging.info("  the first time to load extra images will take long time to convert the file format ...")
            filename = "extra_32x32.mat"
            filepath = maybe_download_and_extract(filename, path, url)
            mat = sio.loadmat(filepath)
            X_extra = mat['X'] / 255.0
            X_extra = np.transpose(X_extra, (3, 0, 1, 2))
            y_extra = np.squeeze(mat['y'], axis=1)
            y_extra[y_extra == 10] = 0
            np.savez(np_file, X=X_extra, y=y_extra)
            del_file(filepath)
        else:
            v = np.load(np_file)
            X_extra = v['X']
            y_extra = v['y']
        # print(X_train.shape, X_extra.shape)
        logging.info("  adding n_extra {} to n_train {}".format(len(y_extra), len(y_train)))
        t = time.time()
        X_train = np.concatenate((X_train, X_extra), 0)
        y_train = np.concatenate((y_train, y_extra), 0)
        # X_train = np.append(X_train, X_extra, axis=0)
        # y_train = np.append(y_train, y_extra, axis=0)
        logging.info("  added n_extra {} to n_train {} took {}s".format(len(y_extra), len(y_train), time.time() - t))
    else:
        logging.info("  no extra images are included")
    logging.info("  image size: %s n_train: %d n_test: %d" % (str(X_train.shape[1:4]), len(y_train), len(y_test)))
    logging.info("  took: {}s".format(int(time.time() - start_time)))
    return X_train, y_train, X_test, y_test

Response: Load Cropped SVHN dataset.


Instruction:
Can you generate the documentation for the following Python 3 function
def load_ptb_dataset(path='data'):
    path = os.path.join(path, 'ptb')
    logging.info("Load or Download Penn TreeBank (PTB) dataset > {}".format(path))

    # Maybe dowload and uncompress tar, or load exsisting files
    filename = 'simple-examples.tgz'
    url = 'http://www.fit.vutbr.cz/~imikolov/rnnlm/'
    maybe_download_and_extract(filename, path, url, extract=True)

    data_path = os.path.join(path, 'simple-examples', 'data')
    train_path = os.path.join(data_path, "ptb.train.txt")
    valid_path = os.path.join(data_path, "ptb.valid.txt")
    test_path = os.path.join(data_path, "ptb.test.txt")

    word_to_id = nlp.build_vocab(nlp.read_words(train_path))

    train_data = nlp.words_to_word_ids(nlp.read_words(train_path), word_to_id)
    valid_data = nlp.words_to_word_ids(nlp.read_words(valid_path), word_to_id)
    test_data = nlp.words_to_word_ids(nlp.read_words(test_path), word_to_id)
    vocab_size = len(word_to_id)

    # logging.info(nlp.read_words(train_path)) # ... 'according', 'to', 'mr.', '<unk>', '<eos>']
    # logging.info(train_data)                 # ...  214,         5,    23,    1,       2]
    # logging.info(word_to_id)                 # ... 'beyond': 1295, 'anti-nuclear': 9599, 'trouble': 1520, '<eos>': 2 ... }
    # logging.info(vocabulary)                 # 10000
    # exit()
    return train_data, valid_data, test_data, vocab_size

Response: Loads a Penn TreeBank dataset.


Instruction:
Can you tell what is the following Python 3 function doing
def load_matt_mahoney_text8_dataset(path='data'):
    path = os.path.join(path, 'mm_test8')
    logging.info("Load or Download matt_mahoney_text8 Dataset> {}".format(path))

    filename = 'text8.zip'
    url = 'http://mattmahoney.net/dc/'
    maybe_download_and_extract(filename, path, url, expected_bytes=31344016)

    with zipfile.ZipFile(os.path.join(path, filename)) as f:
        word_list = f.read(f.namelist()[0]).split()
        for idx, _ in enumerate(word_list):
            word_list[idx] = word_list[idx].decode()
    return word_list

Response: Load Matt Mahoney s dataset.


Instruction:
How would you code a function in Python 3 to
load IMDB dataset. Parameters ---------- path : str The path that the data is downloaded to, defaults is ``data/imdb/``. nb_words : int Number of words to get. skip_top : int Top most frequent words to ignore (they will appear as oov_char value in the sequence data). maxlen : int Maximum sequence length. Any longer sequence will be truncated. seed : int Seed for reproducible data shuffling. start_char : int The start of a sequence will be marked with this character. Set to 1 because 0 is usually the padding character. oov_char : int Words that were cut out because of the num_words or skip_top limit will be replaced with this character. index_from : int Index actual words with this index and higher. Examples -------- >>> X_train, y_train, X_test, y_test = tl.files.load_imdb_dataset( ... nb_words=20000, test_split=0.2) >>> print('X_train.shape', X_train.shape) (20000,) [[1, 62, 74, ... 1033, 507, 27],[1, 60, 33, ... 13, 1053, 7]..] >>> print('y_train.shape', y_train.shape) (20000,) [1 0 0 ..., 1 0 1] References ----------- - `Modified from keras. <https://github.com/fchollet/keras/blob/master/keras/datasets/imdb.py>`__

Response:
def load_imdb_dataset(
        path='data', nb_words=None, skip_top=0, maxlen=None, test_split=0.2, seed=113, start_char=1, oov_char=2,
        index_from=3
):
    """Load IMDB dataset.

    Parameters
    ----------
    path : str
        The path that the data is downloaded to, defaults is ``data/imdb/``.
    nb_words : int
        Number of words to get.
    skip_top : int
        Top most frequent words to ignore (they will appear as oov_char value in the sequence data).
    maxlen : int
        Maximum sequence length. Any longer sequence will be truncated.
    seed : int
        Seed for reproducible data shuffling.
    start_char : int
        The start of a sequence will be marked with this character. Set to 1 because 0 is usually the padding character.
    oov_char : int
        Words that were cut out because of the num_words or skip_top limit will be replaced with this character.
    index_from : int
        Index actual words with this index and higher.

    Examples
    --------
    >>> X_train, y_train, X_test, y_test = tl.files.load_imdb_dataset(
    ...                                 nb_words=20000, test_split=0.2)
    >>> print('X_train.shape', X_train.shape)
    (20000,)  [[1, 62, 74, ... 1033, 507, 27],[1, 60, 33, ... 13, 1053, 7]..]
    >>> print('y_train.shape', y_train.shape)
    (20000,)  [1 0 0 ..., 1 0 1]

    References
    -----------
    - `Modified from keras. <https://github.com/fchollet/keras/blob/master/keras/datasets/imdb.py>`__

    """
    path = os.path.join(path, 'imdb')

    filename = "imdb.pkl"
    url = 'https://s3.amazonaws.com/text-datasets/'
    maybe_download_and_extract(filename, path, url)

    if filename.endswith(".gz"):
        f = gzip.open(os.path.join(path, filename), 'rb')
    else:
        f = open(os.path.join(path, filename), 'rb')

    X, labels = cPickle.load(f)
    f.close()

    np.random.seed(seed)
    np.random.shuffle(X)
    np.random.seed(seed)
    np.random.shuffle(labels)

    if start_char is not None:
        X = [[start_char] + [w + index_from for w in x] for x in X]
    elif index_from:
        X = [[w + index_from for w in x] for x in X]

    if maxlen:
        new_X = []
        new_labels = []
        for x, y in zip(X, labels):
            if len(x) < maxlen:
                new_X.append(x)
                new_labels.append(y)
        X = new_X
        labels = new_labels
    if not X:
        raise Exception(
            'After filtering for sequences shorter than maxlen=' + str(maxlen) + ', no sequence was kept. '
            'Increase maxlen.'
        )
    if not nb_words:
        nb_words = max([max(x) for x in X])

    # by convention, use 2 as OOV word
    # reserve 'index_from' (=3 by default) characters: 0 (padding), 1 (start), 2 (OOV)
    if oov_char is not None:
        X = [[oov_char if (w >= nb_words or w < skip_top) else w for w in x] for x in X]
    else:
        nX = []
        for x in X:
            nx = []
            for w in x:
                if (w >= nb_words or w < skip_top):
                    nx.append(w)
            nX.append(nx)
        X = nX

    X_train = np.array(X[:int(len(X) * (1 - test_split))])
    y_train = np.array(labels[:int(len(X) * (1 - test_split))])

    X_test = np.array(X[int(len(X) * (1 - test_split)):])
    y_test = np.array(labels[int(len(X) * (1 - test_split)):])

    return X_train, y_train, X_test, y_test


Instruction:
Make a summary of the following Python 3 code
def load_nietzsche_dataset(path='data'):
    logging.info("Load or Download nietzsche dataset > {}".format(path))
    path = os.path.join(path, 'nietzsche')

    filename = "nietzsche.txt"
    url = 'https://s3.amazonaws.com/text-datasets/'
    filepath = maybe_download_and_extract(filename, path, url)

    with open(filepath, "r") as f:
        words = f.read()
        return words

Response:
Load Nietzsche dataset.

    Parameters
    ----------
    path : str
        The path that the data is downloaded to, defaults is ``data/nietzsche/``.

    Returns
    --------
    str
        The content.

    Examples
    --------
    >>> see tutorial_generate_text.py
    >>> words = tl.files.load_nietzsche_dataset()
    >>> words = basic_clean_str(words)
    >>> words = words.split()


Instruction:
Implement a function in Python 3 to
load the WMT en - fr dataset.

Response:
def load_wmt_en_fr_dataset(path='data'):
    """Load WMT'15 English-to-French translation dataset.

    It will download the data from the WMT'15 Website (10^9-French-English corpus), and the 2013 news test from the same site as development set.
    Returns the directories of training data and test data.

    Parameters
    ----------
    path : str
        The path that the data is downloaded to, defaults is ``data/wmt_en_fr/``.

    References
    ----------
    - Code modified from /tensorflow/models/rnn/translation/data_utils.py

    Notes
    -----
    Usually, it will take a long time to download this dataset.

    """
    path = os.path.join(path, 'wmt_en_fr')
    # URLs for WMT data.
    _WMT_ENFR_TRAIN_URL = "http://www.statmt.org/wmt10/"
    _WMT_ENFR_DEV_URL = "http://www.statmt.org/wmt15/"

    def gunzip_file(gz_path, new_path):
        """Unzips from gz_path into new_path."""
        logging.info("Unpacking %s to %s" % (gz_path, new_path))
        with gzip.open(gz_path, "rb") as gz_file:
            with open(new_path, "wb") as new_file:
                for line in gz_file:
                    new_file.write(line)

    def get_wmt_enfr_train_set(path):
        """Download the WMT en-fr training corpus to directory unless it's there."""
        filename = "training-giga-fren.tar"
        maybe_download_and_extract(filename, path, _WMT_ENFR_TRAIN_URL, extract=True)
        train_path = os.path.join(path, "giga-fren.release2.fixed")
        gunzip_file(train_path + ".fr.gz", train_path + ".fr")
        gunzip_file(train_path + ".en.gz", train_path + ".en")
        return train_path

    def get_wmt_enfr_dev_set(path):
        """Download the WMT en-fr training corpus to directory unless it's there."""
        filename = "dev-v2.tgz"
        dev_file = maybe_download_and_extract(filename, path, _WMT_ENFR_DEV_URL, extract=False)
        dev_name = "newstest2013"
        dev_path = os.path.join(path, "newstest2013")
        if not (gfile.Exists(dev_path + ".fr") and gfile.Exists(dev_path + ".en")):
            logging.info("Extracting tgz file %s" % dev_file)
            with tarfile.open(dev_file, "r:gz") as dev_tar:
                fr_dev_file = dev_tar.getmember("dev/" + dev_name + ".fr")
                en_dev_file = dev_tar.getmember("dev/" + dev_name + ".en")
                fr_dev_file.name = dev_name + ".fr"  # Extract without "dev/" prefix.
                en_dev_file.name = dev_name + ".en"
                dev_tar.extract(fr_dev_file, path)
                dev_tar.extract(en_dev_file, path)
        return dev_path

    logging.info("Load or Download WMT English-to-French translation > {}".format(path))

    train_path = get_wmt_enfr_train_set(path)
    dev_path = get_wmt_enfr_dev_set(path)

    return train_path, dev_path


Instruction:
How would you explain what the following Python 3 function does
def load_flickr25k_dataset(tag='sky', path="data", n_threads=50, printable=False):
    path = os.path.join(path, 'flickr25k')

    filename = 'mirflickr25k.zip'
    url = 'http://press.liacs.nl/mirflickr/mirflickr25k/'

    # download dataset
    if folder_exists(os.path.join(path, "mirflickr")) is False:
        logging.info("[*] Flickr25k is nonexistent in {}".format(path))
        maybe_download_and_extract(filename, path, url, extract=True)
        del_file(os.path.join(path, filename))

    # return images by the given tag.
    # 1. image path list
    folder_imgs = os.path.join(path, "mirflickr")
    path_imgs = load_file_list(path=folder_imgs, regx='\\.jpg', printable=False)
    path_imgs.sort(key=natural_keys)

    # 2. tag path list
    folder_tags = os.path.join(path, "mirflickr", "meta", "tags")
    path_tags = load_file_list(path=folder_tags, regx='\\.txt', printable=False)
    path_tags.sort(key=natural_keys)

    # 3. select images
    if tag is None:
        logging.info("[Flickr25k] reading all images")
    else:
        logging.info("[Flickr25k] reading images with tag: {}".format(tag))
    images_list = []
    for idx, _v in enumerate(path_tags):
        tags = read_file(os.path.join(folder_tags, path_tags[idx])).split('\n')
        # logging.info(idx+1, tags)
        if tag is None or tag in tags:
            images_list.append(path_imgs[idx])

    images = visualize.read_images(images_list, folder_imgs, n_threads=n_threads, printable=printable)
    return images

Response: Load a list of images by a given tag from Flick25k dataset.


Instruction:
How would you implement a function in Python 3 that
loads Flick1M dataset. Returns a list of images by a given tag from Flickr1M dataset, it will download Flickr1M from `the official website <http://press.liacs.nl/mirflickr/mirdownload.html>`__ at the first time you use it. Parameters ------------ tag : str or None What images to return. - If you want to get images with tag, use string like 'dog', 'red', see `Flickr Search <https://www.flickr.com/search/>`__. - If you want to get all images, set to ``None``. size : int integer between 1 to 10. 1 means 100k images ... 5 means 500k images, 10 means all 1 million images. Default is 10. path : str The path that the data is downloaded to, defaults is ``data/flickr25k/``. n_threads : int The number of thread to read image. printable : boolean Whether to print infomation when reading images, default is ``False``. Examples ---------- Use 200k images >>> images = tl.files.load_flickr1M_dataset(tag='zebra', size=2) Use 1 Million images >>> images = tl.files.load_flickr1M_dataset(tag='zebra')

Response:
def load_flickr1M_dataset(tag='sky', size=10, path="data", n_threads=50, printable=False):
    """Load Flick1M dataset.

    Returns a list of images by a given tag from Flickr1M dataset,
    it will download Flickr1M from `the official website <http://press.liacs.nl/mirflickr/mirdownload.html>`__
    at the first time you use it.

    Parameters
    ------------
    tag : str or None
        What images to return.
            - If you want to get images with tag, use string like 'dog', 'red', see `Flickr Search <https://www.flickr.com/search/>`__.
            - If you want to get all images, set to ``None``.

    size : int
        integer between 1 to 10. 1 means 100k images ... 5 means 500k images, 10 means all 1 million images. Default is 10.
    path : str
        The path that the data is downloaded to, defaults is ``data/flickr25k/``.
    n_threads : int
        The number of thread to read image.
    printable : boolean
        Whether to print infomation when reading images, default is ``False``.

    Examples
    ----------
    Use 200k images

    >>> images = tl.files.load_flickr1M_dataset(tag='zebra', size=2)

    Use 1 Million images

    >>> images = tl.files.load_flickr1M_dataset(tag='zebra')

    """
    path = os.path.join(path, 'flickr1M')
    logging.info("[Flickr1M] using {}% of images = {}".format(size * 10, size * 100000))
    images_zip = [
        'images0.zip', 'images1.zip', 'images2.zip', 'images3.zip', 'images4.zip', 'images5.zip', 'images6.zip',
        'images7.zip', 'images8.zip', 'images9.zip'
    ]
    tag_zip = 'tags.zip'
    url = 'http://press.liacs.nl/mirflickr/mirflickr1m/'

    # download dataset
    for image_zip in images_zip[0:size]:
        image_folder = image_zip.split(".")[0]
        # logging.info(path+"/"+image_folder)
        if folder_exists(os.path.join(path, image_folder)) is False:
            # logging.info(image_zip)
            logging.info("[Flickr1M] {} is missing in {}".format(image_folder, path))
            maybe_download_and_extract(image_zip, path, url, extract=True)
            del_file(os.path.join(path, image_zip))
            # os.system("mv {} {}".format(os.path.join(path, 'images'), os.path.join(path, image_folder)))
            shutil.move(os.path.join(path, 'images'), os.path.join(path, image_folder))
        else:
            logging.info("[Flickr1M] {} exists in {}".format(image_folder, path))

    # download tag
    if folder_exists(os.path.join(path, "tags")) is False:
        logging.info("[Flickr1M] tag files is nonexistent in {}".format(path))
        maybe_download_and_extract(tag_zip, path, url, extract=True)
        del_file(os.path.join(path, tag_zip))
    else:
        logging.info("[Flickr1M] tags exists in {}".format(path))

    # 1. image path list
    images_list = []
    images_folder_list = []
    for i in range(0, size):
        images_folder_list += load_folder_list(path=os.path.join(path, 'images%d' % i))
    images_folder_list.sort(key=lambda s: int(s.split('/')[-1]))  # folder/images/ddd

    for folder in images_folder_list[0:size * 10]:
        tmp = load_file_list(path=folder, regx='\\.jpg', printable=False)
        tmp.sort(key=lambda s: int(s.split('.')[-2]))  # ddd.jpg
        images_list.extend([os.path.join(folder, x) for x in tmp])

    # 2. tag path list
    tag_list = []
    tag_folder_list = load_folder_list(os.path.join(path, "tags"))

    # tag_folder_list.sort(key=lambda s: int(s.split("/")[-1]))  # folder/images/ddd
    tag_folder_list.sort(key=lambda s: int(os.path.basename(s)))

    for folder in tag_folder_list[0:size * 10]:
        tmp = load_file_list(path=folder, regx='\\.txt', printable=False)
        tmp.sort(key=lambda s: int(s.split('.')[-2]))  # ddd.txt
        tmp = [os.path.join(folder, s) for s in tmp]
        tag_list += tmp

    # 3. select images
    logging.info("[Flickr1M] searching tag: {}".format(tag))
    select_images_list = []
    for idx, _val in enumerate(tag_list):
        tags = read_file(tag_list[idx]).split('\n')
        if tag in tags:
            select_images_list.append(images_list[idx])

    logging.info("[Flickr1M] reading images with tag: {}".format(tag))
    images = visualize.read_images(select_images_list, '', n_threads=n_threads, printable=printable)
    return images


Instruction:
Can you tell what is the following Python 3 function doing
def load_cyclegan_dataset(filename='summer2winter_yosemite', path='data'):
    path = os.path.join(path, 'cyclegan')
    url = 'https://people.eecs.berkeley.edu/~taesung_park/CycleGAN/datasets/'

    if folder_exists(os.path.join(path, filename)) is False:
        logging.info("[*] {} is nonexistent in {}".format(filename, path))
        maybe_download_and_extract(filename + '.zip', path, url, extract=True)
        del_file(os.path.join(path, filename + '.zip'))

    def load_image_from_folder(path):
        path_imgs = load_file_list(path=path, regx='\\.jpg', printable=False)
        return visualize.read_images(path_imgs, path=path, n_threads=10, printable=False)

    im_train_A = load_image_from_folder(os.path.join(path, filename, "trainA"))
    im_train_B = load_image_from_folder(os.path.join(path, filename, "trainB"))
    im_test_A = load_image_from_folder(os.path.join(path, filename, "testA"))
    im_test_B = load_image_from_folder(os.path.join(path, filename, "testB"))

    def if_2d_to_3d(images):  # [h, w] --> [h, w, 3]
        for i, _v in enumerate(images):
            if len(images[i].shape) == 2:
                images[i] = images[i][:, :, np.newaxis]
                images[i] = np.tile(images[i], (1, 1, 3))
        return images

    im_train_A = if_2d_to_3d(im_train_A)
    im_train_B = if_2d_to_3d(im_train_B)
    im_test_A = if_2d_to_3d(im_test_A)
    im_test_B = if_2d_to_3d(im_test_B)

    return im_train_A, im_train_B, im_test_A, im_test_B

Response: Loads the CycleGAN dataset from the CycleGAN s database.


Instruction:
Can you tell what is the following Python 3 function doing
def download_file_from_google_drive(ID, destination):

    def save_response_content(response, destination, chunk_size=32 * 1024):
        total_size = int(response.headers.get('content-length', 0))
        with open(destination, "wb") as f:
            for chunk in tqdm(response.iter_content(chunk_size), total=total_size, unit='B', unit_scale=True,
                              desc=destination):
                if chunk:  # filter out keep-alive new chunks
                    f.write(chunk)

    def get_confirm_token(response):
        for key, value in response.cookies.items():
            if key.startswith('download_warning'):
                return value
        return None

    URL = "https://docs.google.com/uc?export=download"
    session = requests.Session()

    response = session.get(URL, params={'id': ID}, stream=True)
    token = get_confirm_token(response)

    if token:
        params = {'id': ID, 'confirm': token}
        response = session.get(URL, params=params, stream=True)
    save_response_content(response, destination)

Response: Download file from Google Drive.


Instruction:
Can you implement a function in Python 3 that
loads CelebA dataset Return a list of image path. Parameters ----------- path : str The path that the data is downloaded to, defaults is ``data/celebA/``.

Response:
def load_celebA_dataset(path='data'):
    """Load CelebA dataset

    Return a list of image path.

    Parameters
    -----------
    path : str
        The path that the data is downloaded to, defaults is ``data/celebA/``.

    """
    data_dir = 'celebA'
    filename, drive_id = "img_align_celeba.zip", "0B7EVK8r0v71pZjFTYXZWM3FlRnM"
    save_path = os.path.join(path, filename)
    image_path = os.path.join(path, data_dir)
    if os.path.exists(image_path):
        logging.info('[*] {} already exists'.format(save_path))
    else:
        exists_or_mkdir(path)
        download_file_from_google_drive(drive_id, save_path)
        zip_dir = ''
        with zipfile.ZipFile(save_path) as zf:
            zip_dir = zf.namelist()[0]
            zf.extractall(path)
        os.remove(save_path)
        os.rename(os.path.join(path, zip_dir), image_path)

    data_files = load_file_list(path=image_path, regx='\\.jpg', printable=False)
    for i, _v in enumerate(data_files):
        data_files[i] = os.path.join(image_path, data_files[i])
    return data_files


Instruction:
Write a Python 3 script to
save parameters and the file name save parameters into. npz file.

Response:
def save_npz(save_list=None, name='model.npz', sess=None):
    """Input parameters and the file name, save parameters into .npz file. Use tl.utils.load_npz() to restore.

    Parameters
    ----------
    save_list : list of tensor
        A list of parameters (tensor) to be saved.
    name : str
        The name of the `.npz` file.
    sess : None or Session
        Session may be required in some case.

    Examples
    --------
    Save model to npz

    >>> tl.files.save_npz(network.all_params, name='model.npz', sess=sess)

    Load model from npz (Method 1)

    >>> load_params = tl.files.load_npz(name='model.npz')
    >>> tl.files.assign_params(sess, load_params, network)

    Load model from npz (Method 2)

    >>> tl.files.load_and_assign_npz(sess=sess, name='model.npz', network=network)

    Notes
    -----
    If you got session issues, you can change the value.eval() to value.eval(session=sess)

    References
    ----------
    `Saving dictionary using numpy <http://stackoverflow.com/questions/22315595/saving-dictionary-of-header-information-using-numpy-savez>`__

    """
    logging.info("[*] Saving TL params into %s" % name)
    if save_list is None:
        save_list = []

    save_list_var = []
    if sess:
        save_list_var = sess.run(save_list)
    else:
        try:
            save_list_var.extend([v.eval() for v in save_list])
        except Exception:
            logging.info(
                " Fail to save model, Hint: pass the session into this function, tl.files.save_npz(network.all_params, name='model.npz', sess=sess)"
            )
    np.savez(name, params=save_list_var)
    save_list_var = None
    del save_list_var
    logging.info("[*] Saved")


Instruction:
Explain what the following Python 3 code does
def load_npz(path='', name='model.npz'):
    d = np.load(os.path.join(path, name))
    return d['params']

Response: Loads the parameters of a Model saved by tl. files. save_npz.


Instruction:
How would you code a function in Python 3 to
assign the given parameters to the TensorFlow Session.

Response:
def assign_params(sess, params, network):
    """Assign the given parameters to the TensorLayer network.

    Parameters
    ----------
    sess : Session
        TensorFlow Session.
    params : list of array
        A list of parameters (array) in order.
    network : :class:`Layer`
        The network to be assigned.

    Returns
    --------
    list of operations
        A list of tf ops in order that assign params. Support sess.run(ops) manually.

    Examples
    --------
    - See ``tl.files.save_npz``

    References
    ----------
    - `Assign value to a TensorFlow variable <http://stackoverflow.com/questions/34220532/how-to-assign-value-to-a-tensorflow-variable>`__

    """
    ops = []
    for idx, param in enumerate(params):
        ops.append(network.all_params[idx].assign(param))
    if sess is not None:
        sess.run(ops)
    return ops


Instruction:
Write a Python 3 function for
loading a model from npz and assign it to a network.

Response:
def load_and_assign_npz(sess=None, name=None, network=None):
    """Load model from npz and assign to a network.

    Parameters
    -------------
    sess : Session
        TensorFlow Session.
    name : str
        The name of the `.npz` file.
    network : :class:`Layer`
        The network to be assigned.

    Returns
    --------
    False or network
        Returns False, if the model is not exist.

    Examples
    --------
    - See ``tl.files.save_npz``

    """
    if network is None:
        raise ValueError("network is None.")
    if sess is None:
        raise ValueError("session is None.")
    if not os.path.exists(name):
        logging.error("file {} doesn't exist.".format(name))
        return False
    else:
        params = load_npz(name=name)
        assign_params(sess, params, network)
        logging.info("[*] Load {} SUCCESS!".format(name))
        return network


Instruction:
Can you write a function in Python 3 where it
saves parameters and the file name into. npz file.

Response:
def save_npz_dict(save_list=None, name='model.npz', sess=None):
    """Input parameters and the file name, save parameters as a dictionary into .npz file.

    Use ``tl.files.load_and_assign_npz_dict()`` to restore.

    Parameters
    ----------
    save_list : list of parameters
        A list of parameters (tensor) to be saved.
    name : str
        The name of the `.npz` file.
    sess : Session
        TensorFlow Session.

    """
    if sess is None:
        raise ValueError("session is None.")
    if save_list is None:
        save_list = []

    save_list_names = [tensor.name for tensor in save_list]
    save_list_var = sess.run(save_list)
    save_var_dict = {save_list_names[idx]: val for idx, val in enumerate(save_list_var)}
    np.savez(name, **save_var_dict)
    save_list_var = None
    save_var_dict = None
    del save_list_var
    del save_var_dict
    logging.info("[*] Model saved in npz_dict %s" % name)


Instruction:
How would you code a function in Python 3 to
load and assign the parameters saved by save_npz_dict.

Response:
def load_and_assign_npz_dict(name='model.npz', sess=None):
    """Restore the parameters saved by ``tl.files.save_npz_dict()``.

    Parameters
    ----------
    name : str
        The name of the `.npz` file.
    sess : Session
        TensorFlow Session.

    """
    if sess is None:
        raise ValueError("session is None.")

    if not os.path.exists(name):
        logging.error("file {} doesn't exist.".format(name))
        return False

    params = np.load(name)
    if len(params.keys()) != len(set(params.keys())):
        raise Exception("Duplication in model npz_dict %s" % name)
    ops = list()
    for key in params.keys():
        try:
            # tensor = tf.get_default_graph().get_tensor_by_name(key)
            # varlist = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope=key)
            varlist = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope=key)
            if len(varlist) > 1:
                raise Exception("[!] Multiple candidate variables to be assigned for name %s" % key)
            elif len(varlist) == 0:
                raise KeyError
            else:
                ops.append(varlist[0].assign(params[key]))
                logging.info("[*] params restored: %s" % key)
        except KeyError:
            logging.info("[!] Warning: Tensor named %s not found in network." % key)

    sess.run(ops)
    logging.info("[*] Model restored from npz_dict %s" % name)


Instruction:
Explain what the following Python 3 code does
def save_ckpt(
        sess=None, mode_name='model.ckpt', save_dir='checkpoint', var_list=None, global_step=None, printable=False
):
    """Save parameters into `ckpt` file.

    Parameters
    ------------
    sess : Session
        TensorFlow Session.
    mode_name : str
        The name of the model, default is ``model.ckpt``.
    save_dir : str
        The path / file directory to the `ckpt`, default is ``checkpoint``.
    var_list : list of tensor
        The parameters / variables (tensor) to be saved. If empty, save all global variables (default).
    global_step : int or None
        Step number.
    printable : boolean
        Whether to print all parameters information.

    See Also
    --------
    load_ckpt

    """
    if sess is None:
        raise ValueError("session is None.")
    if var_list is None:
        var_list = []

    ckpt_file = os.path.join(save_dir, mode_name)
    if var_list == []:
        var_list = tf.global_variables()

    logging.info("[*] save %s n_params: %d" % (ckpt_file, len(var_list)))

    if printable:
        for idx, v in enumerate(var_list):
            logging.info("  param {:3}: {:15}   {}".format(idx, v.name, str(v.get_shape())))

    saver = tf.train.Saver(var_list)
    saver.save(sess, ckpt_file, global_step=global_step)

Response: Save parameters into CKPT file.


Instruction:
How would you implement a function in Python 3 that
loads parameters from the CKPT file.

Response:
def load_ckpt(sess=None, mode_name='model.ckpt', save_dir='checkpoint', var_list=None, is_latest=True, printable=False):
    """Load parameters from `ckpt` file.

    Parameters
    ------------
    sess : Session
        TensorFlow Session.
    mode_name : str
        The name of the model, default is ``model.ckpt``.
    save_dir : str
        The path / file directory to the `ckpt`, default is ``checkpoint``.
    var_list : list of tensor
        The parameters / variables (tensor) to be saved. If empty, save all global variables (default).
    is_latest : boolean
        Whether to load the latest `ckpt`, if False, load the `ckpt` with the name of ```mode_name``.
    printable : boolean
        Whether to print all parameters information.

    Examples
    ----------
    - Save all global parameters.

    >>> tl.files.save_ckpt(sess=sess, mode_name='model.ckpt', save_dir='model', printable=True)

    - Save specific parameters.

    >>> tl.files.save_ckpt(sess=sess, mode_name='model.ckpt', var_list=net.all_params, save_dir='model', printable=True)

    - Load latest ckpt.

    >>> tl.files.load_ckpt(sess=sess, var_list=net.all_params, save_dir='model', printable=True)

    - Load specific ckpt.

    >>> tl.files.load_ckpt(sess=sess, mode_name='model.ckpt', var_list=net.all_params, save_dir='model', is_latest=False, printable=True)

    """
    if sess is None:
        raise ValueError("session is None.")
    if var_list is None:
        var_list = []

    if is_latest:
        ckpt_file = tf.train.latest_checkpoint(save_dir)
    else:
        ckpt_file = os.path.join(save_dir, mode_name)

    if not var_list:
        var_list = tf.global_variables()

    logging.info("[*] load %s n_params: %d" % (ckpt_file, len(var_list)))

    if printable:
        for idx, v in enumerate(var_list):
            logging.info("  param {:3}: {:15}   {}".format(idx, v.name, str(v.get_shape())))

    try:
        saver = tf.train.Saver(var_list)
        saver.restore(sess, ckpt_file)
    except Exception as e:
        logging.info(e)
        logging.info("[*] load ckpt fail ...")


Instruction:
Write a Python 3 function for
loading any. npy file.

Response:
def load_npy_to_any(path='', name='file.npy'):
    """Load `.npy` file.

    Parameters
    ------------
    path : str
        Path to the file (optional).
    name : str
        File name.

    Examples
    ---------
    - see tl.files.save_any_to_npy()

    """
    file_path = os.path.join(path, name)
    try:
        return np.load(file_path).item()
    except Exception:
        return np.load(file_path)
    raise Exception("[!] Fail to load %s" % file_path)


Instruction:
Can you generate the documentation for the following Python 3 function
def load_folder_list(path=""):
    return [os.path.join(path, o) for o in os.listdir(path) if os.path.isdir(os.path.join(path, o))]

Response: Return a list of folder names in a folder by given a folder path.


Instruction:
How would you code a function in Python 3 to
check a folder by given name and creates it if it does not exist.

Response:
def exists_or_mkdir(path, verbose=True):
    """Check a folder by given name, if not exist, create the folder and return False,
    if directory exists, return True.

    Parameters
    ----------
    path : str
        A folder path.
    verbose : boolean
        If True (default), prints results.

    Returns
    --------
    boolean
        True if folder already exist, otherwise, returns False and create the folder.

    Examples
    --------
    >>> tl.files.exists_or_mkdir("checkpoints/train")

    """
    if not os.path.exists(path):
        if verbose:
            logging.info("[*] creates %s ..." % path)
        os.makedirs(path)
        return False
    else:
        if verbose:
            logging.info("[!] %s exists ..." % path)
        return True


Instruction:
Write a Python 3 function for
downloading and extract the file from the given URL.

Response:
def maybe_download_and_extract(filename, working_directory, url_source, extract=False, expected_bytes=None):
    """Checks if file exists in working_directory otherwise tries to dowload the file,
    and optionally also tries to extract the file if format is ".zip" or ".tar"

    Parameters
    -----------
    filename : str
        The name of the (to be) dowloaded file.
    working_directory : str
        A folder path to search for the file in and dowload the file to
    url : str
        The URL to download the file from
    extract : boolean
        If True, tries to uncompress the dowloaded file is ".tar.gz/.tar.bz2" or ".zip" file, default is False.
    expected_bytes : int or None
        If set tries to verify that the downloaded file is of the specified size, otherwise raises an Exception, defaults is None which corresponds to no check being performed.

    Returns
    ----------
    str
        File path of the dowloaded (uncompressed) file.

    Examples
    --------
    >>> down_file = tl.files.maybe_download_and_extract(filename='train-images-idx3-ubyte.gz',
    ...                                            working_directory='data/',
    ...                                            url_source='http://yann.lecun.com/exdb/mnist/')
    >>> tl.files.maybe_download_and_extract(filename='ADEChallengeData2016.zip',
    ...                                             working_directory='data/',
    ...                                             url_source='http://sceneparsing.csail.mit.edu/data/',
    ...                                             extract=True)

    """

    # We first define a download function, supporting both Python 2 and 3.
    def _download(filename, working_directory, url_source):

        progress_bar = progressbar.ProgressBar()

        def _dlProgress(count, blockSize, totalSize, pbar=progress_bar):
            if (totalSize != 0):

                if not pbar.max_value:
                    totalBlocks = math.ceil(float(totalSize) / float(blockSize))
                    pbar.max_value = int(totalBlocks)

                pbar.update(count, force=True)

        filepath = os.path.join(working_directory, filename)

        logging.info('Downloading %s...\n' % filename)

        urlretrieve(url_source + filename, filepath, reporthook=_dlProgress)

    exists_or_mkdir(working_directory, verbose=False)
    filepath = os.path.join(working_directory, filename)

    if not os.path.exists(filepath):

        _download(filename, working_directory, url_source)
        statinfo = os.stat(filepath)
        logging.info('Succesfully downloaded %s %s bytes.' % (filename, statinfo.st_size))  # , 'bytes.')
        if (not (expected_bytes is None) and (expected_bytes != statinfo.st_size)):
            raise Exception('Failed to verify ' + filename + '. Can you get to it with a browser?')
        if (extract):
            if tarfile.is_tarfile(filepath):
                logging.info('Trying to extract tar file')
                tarfile.open(filepath, 'r').extractall(working_directory)
                logging.info('... Success!')
            elif zipfile.is_zipfile(filepath):
                logging.info('Trying to extract zip file')
                with zipfile.ZipFile(filepath) as zf:
                    zf.extractall(working_directory)
                logging.info('... Success!')
            else:
                logging.info("Unknown compression_format only .tar.gz/.tar.bz2/.tar and .zip supported")
    return filepath


Instruction:
Implement a Python 3 function for
sorting list of string with number in human order.

Response:
def natural_keys(text):
    """Sort list of string with number in human order.

    Examples
    ----------
    >>> l = ['im1.jpg', 'im31.jpg', 'im11.jpg', 'im21.jpg', 'im03.jpg', 'im05.jpg']
    >>> l.sort(key=tl.files.natural_keys)
    ['im1.jpg', 'im03.jpg', 'im05', 'im11.jpg', 'im21.jpg', 'im31.jpg']
    >>> l.sort() # that is what we dont want
    ['im03.jpg', 'im05', 'im1.jpg', 'im11.jpg', 'im21.jpg', 'im31.jpg']

    References
    ----------
    - `link <http://nedbatchelder.com/blog/200712/human_sorting.html>`__

    """

    # - alist.sort(key=natural_keys) sorts in human order
    # http://nedbatchelder.com/blog/200712/human_sorting.html
    # (See Toothy's implementation in the comments)
    def atoi(text):
        return int(text) if text.isdigit() else text

    return [atoi(c) for c in re.split('(\d+)', text)]


Instruction:
Explain what the following Python 3 code does
def npz_to_W_pdf(path=None, regx='w1pre_[0-9]+\.(npz)'):
    file_list = load_file_list(path=path, regx=regx)
    for f in file_list:
        W = load_npz(path, f)[0]
        logging.info("%s --> %s" % (f, f.split('.')[0] + '.pdf'))
        visualize.draw_weights(W, second=10, saveable=True, name=f.split('.')[0], fig_idx=2012)

Response: r Converts the first weight matrix of. npz file to. pdf by using tl. visualize. W.


Instruction:
Can you write a function in Python 3 where it
processes a batch of data by given function by threading.

Response:
def threading_data(data=None, fn=None, thread_count=None, **kwargs):
    """Process a batch of data by given function by threading.

    Usually be used for data augmentation.

    Parameters
    -----------
    data : numpy.array or others
        The data to be processed.
    thread_count : int
        The number of threads to use.
    fn : function
        The function for data processing.
    more args : the args for `fn`
        Ssee Examples below.

    Examples
    --------
    Process images.

    >>> images, _, _, _ = tl.files.load_cifar10_dataset(shape=(-1, 32, 32, 3))
    >>> images = tl.prepro.threading_data(images[0:32], tl.prepro.zoom, zoom_range=[0.5, 1])

    Customized image preprocessing function.

    >>> def distort_img(x):
    >>>     x = tl.prepro.flip_axis(x, axis=0, is_random=True)
    >>>     x = tl.prepro.flip_axis(x, axis=1, is_random=True)
    >>>     x = tl.prepro.crop(x, 100, 100, is_random=True)
    >>>     return x
    >>> images = tl.prepro.threading_data(images, distort_img)

    Process images and masks together (Usually be used for image segmentation).

    >>> X, Y --> [batch_size, row, col, 1]
    >>> data = tl.prepro.threading_data([_ for _ in zip(X, Y)], tl.prepro.zoom_multi, zoom_range=[0.5, 1], is_random=True)
    data --> [batch_size, 2, row, col, 1]
    >>> X_, Y_ = data.transpose((1,0,2,3,4))
    X_, Y_ --> [batch_size, row, col, 1]
    >>> tl.vis.save_image(X_, 'images.png')
    >>> tl.vis.save_image(Y_, 'masks.png')

    Process images and masks together by using ``thread_count``.

    >>> X, Y --> [batch_size, row, col, 1]
    >>> data = tl.prepro.threading_data(X, tl.prepro.zoom_multi, 8, zoom_range=[0.5, 1], is_random=True)
    data --> [batch_size, 2, row, col, 1]
    >>> X_, Y_ = data.transpose((1,0,2,3,4))
    X_, Y_ --> [batch_size, row, col, 1]
    >>> tl.vis.save_image(X_, 'after.png')
    >>> tl.vis.save_image(Y_, 'before.png')

    Customized function for processing images and masks together.

    >>> def distort_img(data):
    >>>    x, y = data
    >>>    x, y = tl.prepro.flip_axis_multi([x, y], axis=0, is_random=True)
    >>>    x, y = tl.prepro.flip_axis_multi([x, y], axis=1, is_random=True)
    >>>    x, y = tl.prepro.crop_multi([x, y], 100, 100, is_random=True)
    >>>    return x, y

    >>> X, Y --> [batch_size, row, col, channel]
    >>> data = tl.prepro.threading_data([_ for _ in zip(X, Y)], distort_img)
    >>> X_, Y_ = data.transpose((1,0,2,3,4))

    Returns
    -------
    list or numpyarray
        The processed results.

    References
    ----------
    - `python queue <https://pymotw.com/2/Queue/index.html#module-Queue>`__
    - `run with limited queue <http://effbot.org/librarybook/queue.htm>`__

    """

    def apply_fn(results, i, data, kwargs):
        results[i] = fn(data, **kwargs)

    if thread_count is None:
        results = [None] * len(data)
        threads = []
        # for i in range(len(data)):
        #     t = threading.Thread(name='threading_and_return', target=apply_fn, args=(results, i, data[i], kwargs))
        for i, d in enumerate(data):
            t = threading.Thread(name='threading_and_return', target=apply_fn, args=(results, i, d, kwargs))
            t.start()
            threads.append(t)
    else:
        divs = np.linspace(0, len(data), thread_count + 1)
        divs = np.round(divs).astype(int)
        results = [None] * thread_count
        threads = []
        for i in range(thread_count):
            t = threading.Thread(
                name='threading_and_return', target=apply_fn, args=(results, i, data[divs[i]:divs[i + 1]], kwargs)
            )
            t.start()
            threads.append(t)

    for t in threads:
        t.join()

    if thread_count is None:
        try:
            return np.asarray(results)
        except Exception:
            return results
    else:
        return np.concatenate(results)


Instruction:
How would you code a function in Python 3 to
create an affine transform matrix for image rotation.

Response:
def affine_rotation_matrix(angle=(-20, 20)):
    """Create an affine transform matrix for image rotation.
    NOTE: In OpenCV, x is width and y is height.

    Parameters
    -----------
    angle : int/float or tuple of two int/float
        Degree to rotate, usually -180 ~ 180.
            - int/float, a fixed angle.
            - tuple of 2 floats/ints, randomly sample a value as the angle between these 2 values.

    Returns
    -------
    numpy.array
        An affine transform matrix.

    """
    if isinstance(angle, tuple):
        theta = np.pi / 180 * np.random.uniform(angle[0], angle[1])
    else:
        theta = np.pi / 180 * angle
    rotation_matrix = np.array([[np.cos(theta), np.sin(theta), 0], \
                                [-np.sin(theta), np.cos(theta), 0], \
                                [0, 0, 1]])
    return rotation_matrix


Instruction:
Write a Python 3 script for
creating an affine transformation matrix for image horizontal flipping.

Response:
def affine_horizontal_flip_matrix(prob=0.5):
    """Create an affine transformation matrix for image horizontal flipping.
    NOTE: In OpenCV, x is width and y is height.

    Parameters
    ----------
    prob : float
        Probability to flip the image. 1.0 means always flip.

    Returns
    -------
    numpy.array
        An affine transform matrix.

    """
    factor = np.random.uniform(0, 1)
    if prob >= factor:
        filp_matrix = np.array([[ -1. , 0., 0. ], \
              [ 0., 1., 0. ], \
              [ 0., 0., 1. ]])
        return filp_matrix
    else:
        filp_matrix = np.array([[ 1. , 0., 0. ], \
              [ 0., 1., 0. ], \
              [ 0., 0., 1. ]])
        return filp_matrix


Instruction:
Make a summary of the following Python 3 code
def affine_vertical_flip_matrix(prob=0.5):
    factor = np.random.uniform(0, 1)
    if prob >= factor:
        filp_matrix = np.array([[ 1. , 0., 0. ], \
              [ 0., -1., 0. ], \
              [ 0., 0., 1. ]])
        return filp_matrix
    else:
        filp_matrix = np.array([[ 1. , 0., 0. ], \
              [ 0., 1., 0. ], \
              [ 0., 0., 1. ]])
        return filp_matrix

Response: Create an affine transformation for image vertical flipping.


Instruction:
Write a Python 3 function that can
create an affine transform matrix for image shifting.

Response:
def affine_shift_matrix(wrg=(-0.1, 0.1), hrg=(-0.1, 0.1), w=200, h=200):
    """Create an affine transform matrix for image shifting.
    NOTE: In OpenCV, x is width and y is height.

    Parameters
    -----------
    wrg : float or tuple of floats
        Range to shift on width axis, -1 ~ 1.
            - float, a fixed distance.
            - tuple of 2 floats, randomly sample a value as the distance between these 2 values.
    hrg : float or tuple of floats
        Range to shift on height axis, -1 ~ 1.
            - float, a fixed distance.
            - tuple of 2 floats, randomly sample a value as the distance between these 2 values.
    w, h : int
        The width and height of the image.

    Returns
    -------
    numpy.array
        An affine transform matrix.

    """
    if isinstance(wrg, tuple):
        tx = np.random.uniform(wrg[0], wrg[1]) * w
    else:
        tx = wrg * w
    if isinstance(hrg, tuple):
        ty = np.random.uniform(hrg[0], hrg[1]) * h
    else:
        ty = hrg * h
    shift_matrix = np.array([[1, 0, tx], \
                        [0, 1, ty], \
                        [0, 0, 1]])
    return shift_matrix


Instruction:
Here you have a function in Python 3, explain what it does
def affine_shear_matrix(x_shear=(-0.1, 0.1), y_shear=(-0.1, 0.1)):
    # if len(shear) != 2:
    #     raise AssertionError(
    #         "shear should be tuple of 2 floats, or you want to use tl.prepro.shear rather than tl.prepro.shear2 ?"
    #     )
    # if isinstance(shear, tuple):
    #     shear = list(shear)
    # if is_random:
    #     shear[0] = np.random.uniform(-shear[0], shear[0])
    #     shear[1] = np.random.uniform(-shear[1], shear[1])
    if isinstance(x_shear, tuple):
        x_shear = np.random.uniform(x_shear[0], x_shear[1])
    if isinstance(y_shear, tuple):
        y_shear = np.random.uniform(y_shear[0], y_shear[1])

    shear_matrix = np.array([[1, x_shear, 0], \
                            [y_shear, 1, 0], \
                            [0, 0, 1]])
    return shear_matrix

Response: Create an affine transform matrix for image shearing.


Instruction:
Here you have a function in Python 3, explain what it does
def affine_zoom_matrix(zoom_range=(0.8, 1.1)):

    if isinstance(zoom_range, (float, int)):
        scale = zoom_range
    elif isinstance(zoom_range, tuple):
        scale = np.random.uniform(zoom_range[0], zoom_range[1])
    else:
        raise Exception("zoom_range: float or tuple of 2 floats")

    zoom_matrix = np.array([[scale, 0, 0], \
                            [0, scale, 0], \
                            [0, 0, 1]])
    return zoom_matrix

Response: Create an affine transform matrix for zooming and scaling an image s height and width.


Instruction:
Can you implement a function in Python 3 that
gets affine transform matrix for zooming and scaling that height and width are changed independently.

Response:
def affine_respective_zoom_matrix(w_range=0.8, h_range=1.1):
    """Get affine transform matrix for zooming/scaling that height and width are changed independently.
    OpenCV format, x is width.

    Parameters
    -----------
    w_range : float or tuple of 2 floats
        The zooming/scaling ratio of width, greater than 1 means larger.
            - float, a fixed ratio.
            - tuple of 2 floats, randomly sample a value as the ratio between 2 values.
    h_range : float or tuple of 2 floats
        The zooming/scaling ratio of height, greater than 1 means larger.
            - float, a fixed ratio.
            - tuple of 2 floats, randomly sample a value as the ratio between 2 values.

    Returns
    -------
    numpy.array
        An affine transform matrix.

    """

    if isinstance(h_range, (float, int)):
        zy = h_range
    elif isinstance(h_range, tuple):
        zy = np.random.uniform(h_range[0], h_range[1])
    else:
        raise Exception("h_range: float or tuple of 2 floats")

    if isinstance(w_range, (float, int)):
        zx = w_range
    elif isinstance(w_range, tuple):
        zx = np.random.uniform(w_range[0], w_range[1])
    else:
        raise Exception("w_range: float or tuple of 2 floats")

    zoom_matrix = np.array([[zx, 0, 0], \
                            [0, zy, 0], \
                            [0, 0, 1]])
    return zoom_matrix


Instruction:
Can you create a Python 3 function that
converts the matrix from Cartesian coordinates to Image coordinates.

Response:
def transform_matrix_offset_center(matrix, y, x):
    """Convert the matrix from Cartesian coordinates (the origin in the middle of image) to Image coordinates (the origin on the top-left of image).

    Parameters
    ----------
    matrix : numpy.array
        Transform matrix.
    x and y : 2 int
        Size of image.

    Returns
    -------
    numpy.array
        The transform matrix.

    Examples
    --------
    - See ``tl.prepro.rotation``, ``tl.prepro.shear``, ``tl.prepro.zoom``.
    """
    o_x = (x - 1) / 2.0
    o_y = (y - 1) / 2.0
    offset_matrix = np.array([[1, 0, o_x], [0, 1, o_y], [0, 0, 1]])
    reset_matrix = np.array([[1, 0, -o_x], [0, 1, -o_y], [0, 0, 1]])
    transform_matrix = np.dot(np.dot(offset_matrix, matrix), reset_matrix)
    return transform_matrix


Instruction:
Can you generate the documentation for the following Python 3 function
def affine_transform(x, transform_matrix, channel_index=2, fill_mode='nearest', cval=0., order=1):
    # transform_matrix = transform_matrix_offset_center()
    # asdihasid
    # asd

    x = np.rollaxis(x, channel_index, 0)
    final_affine_matrix = transform_matrix[:2, :2]
    final_offset = transform_matrix[:2, 2]
    channel_images = [
        ndi.interpolation.
        affine_transform(x_channel, final_affine_matrix, final_offset, order=order, mode=fill_mode, cval=cval)
        for x_channel in x
    ]
    x = np.stack(channel_images, axis=0)
    x = np.rollaxis(x, 0, channel_index + 1)
    return x

Response: Return transformed images by given an affine matrix in Scipy format.


Instruction:
How would you implement a function in Python 3 that
returns transformed images by given an affine matrix in OpenCV2 format.

Response:
def affine_transform_cv2(x, transform_matrix, flags=None, border_mode='constant'):
    """Return transformed images by given an affine matrix in OpenCV format (x is width). (Powered by OpenCV2, faster than ``tl.prepro.affine_transform``)

    Parameters
    ----------
    x : numpy.array
        An image with dimension of [row, col, channel] (default).
    transform_matrix : numpy.array
        A transform matrix, OpenCV format.
    border_mode : str
        - `constant`, pad the image with a constant value (i.e. black or 0)
        - `replicate`, the row or column at the very edge of the original is replicated to the extra border.

    Examples
    --------
    >>> M_shear = tl.prepro.affine_shear_matrix(intensity=0.2, is_random=False)
    >>> M_zoom = tl.prepro.affine_zoom_matrix(zoom_range=0.8)
    >>> M_combined = M_shear.dot(M_zoom)
    >>> result = tl.prepro.affine_transform_cv2(image, M_combined)
    """
    rows, cols = x.shape[0], x.shape[1]
    if flags is None:
        flags = cv2.INTER_AREA
    if border_mode is 'constant':
        border_mode = cv2.BORDER_CONSTANT
    elif border_mode is 'replicate':
        border_mode = cv2.BORDER_REPLICATE
    else:
        raise Exception("unsupport border_mode, check cv.BORDER_ for more details.")
    return cv2.warpAffine(x, transform_matrix[0:2,:], \
            (cols,rows), flags=flags, borderMode=border_mode)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def affine_transform_keypoints(coords_list, transform_matrix):
    coords_result_list = []
    for coords in coords_list:
        coords = np.asarray(coords)
        coords = coords.transpose([1, 0])
        coords = np.insert(coords, 2, 1, axis=0)
        # print(coords)
        # print(transform_matrix)
        coords_result = np.matmul(transform_matrix, coords)
        coords_result = coords_result[0:2, :].transpose([1, 0])
        coords_result_list.append(coords_result)
    return coords_result_list

Response: Transform keypoint coordinates according to a given affine transform matrix.


Instruction:
Make a summary of the following Python 3 code
def projective_transform_by_points(
        x, src, dst, map_args=None, output_shape=None, order=1, mode='constant', cval=0.0, clip=True,
        preserve_range=False
):
    """Projective transform by given coordinates, usually 4 coordinates.

    see `scikit-image <http://scikit-image.org/docs/dev/auto_examples/applications/plot_geometric.html>`__.

    Parameters
    -----------
    x : numpy.array
        An image with dimension of [row, col, channel] (default).
    src : list or numpy
        The original coordinates, usually 4 coordinates of (width, height).
    dst : list or numpy
        The coordinates after transformation, the number of coordinates is the same with src.
    map_args : dictionary or None
        Keyword arguments passed to inverse map.
    output_shape : tuple of 2 int
        Shape of the output image generated. By default the shape of the input image is preserved. Note that, even for multi-band images, only rows and columns need to be specified.
    order : int
        The order of interpolation. The order has to be in the range 0-5:
            - 0 Nearest-neighbor
            - 1 Bi-linear (default)
            - 2 Bi-quadratic
            - 3 Bi-cubic
            - 4 Bi-quartic
            - 5 Bi-quintic
    mode : str
        One of `constant` (default), `edge`, `symmetric`, `reflect` or `wrap`.
        Points outside the boundaries of the input are filled according to the given mode. Modes match the behaviour of numpy.pad.
    cval : float
        Used in conjunction with mode `constant`, the value outside the image boundaries.
    clip : boolean
        Whether to clip the output to the range of values of the input image. This is enabled by default, since higher order interpolation may produce values outside the given input range.
    preserve_range : boolean
        Whether to keep the original range of values. Otherwise, the input image is converted according to the conventions of img_as_float.

    Returns
    -------
    numpy.array
        A processed image.

    Examples
    --------
    Assume X is an image from CIFAR-10, i.e. shape == (32, 32, 3)

    >>> src = [[0,0],[0,32],[32,0],[32,32]]     # [w, h]
    >>> dst = [[10,10],[0,32],[32,0],[32,32]]
    >>> x = tl.prepro.projective_transform_by_points(X, src, dst)

    References
    -----------
    - `scikit-image : geometric transformations <http://scikit-image.org/docs/dev/auto_examples/applications/plot_geometric.html>`__
    - `scikit-image : examples <http://scikit-image.org/docs/dev/auto_examples/index.html>`__

    """
    if map_args is None:
        map_args = {}
    # if type(src) is list:
    if isinstance(src, list):  # convert to numpy
        src = np.array(src)
    # if type(dst) is list:
    if isinstance(dst, list):
        dst = np.array(dst)
    if np.max(x) > 1:  # convert to [0, 1]
        x = x / 255

    m = transform.ProjectiveTransform()
    m.estimate(dst, src)
    warped = transform.warp(
        x, m, map_args=map_args, output_shape=output_shape, order=order, mode=mode, cval=cval, clip=clip,
        preserve_range=preserve_range
    )
    return warped

Response: Projective transform by given points.


Instruction:
Can you tell what is the following Python 3 function doing
def rotation(
        x, rg=20, is_random=False, row_index=0, col_index=1, channel_index=2, fill_mode='nearest', cval=0., order=1
):
    """Rotate an image randomly or non-randomly.

    Parameters
    -----------
    x : numpy.array
        An image with dimension of [row, col, channel] (default).
    rg : int or float
        Degree to rotate, usually 0 ~ 180.
    is_random : boolean
        If True, randomly rotate. Default is False
    row_index col_index and channel_index : int
        Index of row, col and channel, default (0, 1, 2), for theano (1, 2, 0).
    fill_mode : str
        Method to fill missing pixel, default `nearest`, more options `constant`, `reflect` or `wrap`, see `scipy ndimage affine_transform <https://docs.scipy.org/doc/scipy-0.14.0/reference/generated/scipy.ndimage.interpolation.affine_transform.html>`__
    cval : float
        Value used for points outside the boundaries of the input if mode=`constant`. Default is 0.0
    order : int
        The order of interpolation. The order has to be in the range 0-5. See ``tl.prepro.affine_transform`` and `scipy ndimage affine_transform <https://docs.scipy.org/doc/scipy-0.14.0/reference/generated/scipy.ndimage.interpolation.affine_transform.html>`__

    Returns
    -------
    numpy.array
        A processed image.

    Examples
    ---------
    >>> x --> [row, col, 1]
    >>> x = tl.prepro.rotation(x, rg=40, is_random=False)
    >>> tl.vis.save_image(x, 'im.png')

    """
    if is_random:
        theta = np.pi / 180 * np.random.uniform(-rg, rg)
    else:
        theta = np.pi / 180 * rg
    rotation_matrix = np.array([[np.cos(theta), -np.sin(theta), 0], [np.sin(theta), np.cos(theta), 0], [0, 0, 1]])

    h, w = x.shape[row_index], x.shape[col_index]
    transform_matrix = transform_matrix_offset_center(rotation_matrix, h, w)
    x = affine_transform(x, transform_matrix, channel_index, fill_mode, cval, order)
    return x

Response: Rotate an image randomly or non - randomly.


Instruction:
Make a summary of the following Python 3 code
def crop_multi(x, wrg, hrg, is_random=False, row_index=0, col_index=1):
    h, w = x[0].shape[row_index], x[0].shape[col_index]

    if (h < hrg) or (w < wrg):
        raise AssertionError("The size of cropping should smaller than or equal to the original image")

    if is_random:
        h_offset = int(np.random.uniform(0, h - hrg))
        w_offset = int(np.random.uniform(0, w - wrg))
        results = []
        for data in x:
            results.append(data[h_offset:hrg + h_offset, w_offset:wrg + w_offset])
        return np.asarray(results)
    else:
        # central crop
        h_offset = (h - hrg) / 2
        w_offset = (w - wrg) / 2
        results = []
        for data in x:
            results.append(data[h_offset:h - h_offset, w_offset:w - w_offset])
        return np.asarray(results)

Response:
Randomly or centrally crop multiple images.

    Parameters
    ----------
    x : list of numpy.array
        List of images with dimension of [n_images, row, col, channel] (default).
    others : args
        See ``tl.prepro.crop``.

    Returns
    -------
    numpy.array
        A list of processed images.


Instruction:
Write a Python 3 script for
flipping the axis of an image in a random order.

Response:
def flip_axis(x, axis=1, is_random=False):
    """Flip the axis of an image, such as flip left and right, up and down, randomly or non-randomly,

    Parameters
    ----------
    x : numpy.array
        An image with dimension of [row, col, channel] (default).
    axis : int
        Which axis to flip.
            - 0, flip up and down
            - 1, flip left and right
            - 2, flip channel
    is_random : boolean
        If True, randomly flip. Default is False.

    Returns
    -------
    numpy.array
        A processed image.

    """
    if is_random:
        factor = np.random.uniform(-1, 1)
        if factor > 0:
            x = np.asarray(x).swapaxes(axis, 0)
            x = x[::-1, ...]
            x = x.swapaxes(0, axis)
            return x
        else:
            return x
    else:
        x = np.asarray(x).swapaxes(axis, 0)
        x = x[::-1, ...]
        x = x.swapaxes(0, axis)
        return x


Instruction:
Write a Python 3 script to
flip the axises of multiple images together.

Response:
def flip_axis_multi(x, axis, is_random=False):
    """Flip the axises of multiple images together, such as flip left and right, up and down, randomly or non-randomly,

    Parameters
    -----------
    x : list of numpy.array
        List of images with dimension of [n_images, row, col, channel] (default).
    others : args
        See ``tl.prepro.flip_axis``.

    Returns
    -------
    numpy.array
        A list of processed images.

    """
    if is_random:
        factor = np.random.uniform(-1, 1)
        if factor > 0:
            # x = np.asarray(x).swapaxes(axis, 0)
            # x = x[::-1, ...]
            # x = x.swapaxes(0, axis)
            # return x
            results = []
            for data in x:
                data = np.asarray(data).swapaxes(axis, 0)
                data = data[::-1, ...]
                data = data.swapaxes(0, axis)
                results.append(data)
            return np.asarray(results)
        else:
            return np.asarray(x)
    else:
        # x = np.asarray(x).swapaxes(axis, 0)
        # x = x[::-1, ...]
        # x = x.swapaxes(0, axis)
        # return x
        results = []
        for data in x:
            data = np.asarray(data).swapaxes(axis, 0)
            data = data[::-1, ...]
            data = data.swapaxes(0, axis)
            results.append(data)
        return np.asarray(results)


Instruction:
Can you implement a function in Python 3 that
shifts an image randomly or non - randomly.

Response:
def shift(
        x, wrg=0.1, hrg=0.1, is_random=False, row_index=0, col_index=1, channel_index=2, fill_mode='nearest', cval=0.,
        order=1
):
    """Shift an image randomly or non-randomly.

    Parameters
    -----------
    x : numpy.array
        An image with dimension of [row, col, channel] (default).
    wrg : float
        Percentage of shift in axis x, usually -0.25 ~ 0.25.
    hrg : float
        Percentage of shift in axis y, usually -0.25 ~ 0.25.
    is_random : boolean
        If True, randomly shift. Default is False.
    row_index col_index and channel_index : int
        Index of row, col and channel, default (0, 1, 2), for theano (1, 2, 0).
    fill_mode : str
        Method to fill missing pixel, default `nearest`, more options `constant`, `reflect` or `wrap`, see `scipy ndimage affine_transform <https://docs.scipy.org/doc/scipy-0.14.0/reference/generated/scipy.ndimage.interpolation.affine_transform.html>`__
    cval : float
        Value used for points outside the boundaries of the input if mode='constant'. Default is 0.0.
    order : int
        The order of interpolation. The order has to be in the range 0-5. See ``tl.prepro.affine_transform`` and `scipy ndimage affine_transform <https://docs.scipy.org/doc/scipy-0.14.0/reference/generated/scipy.ndimage.interpolation.affine_transform.html>`__

    Returns
    -------
    numpy.array
        A processed image.

    """
    h, w = x.shape[row_index], x.shape[col_index]
    if is_random:
        tx = np.random.uniform(-hrg, hrg) * h
        ty = np.random.uniform(-wrg, wrg) * w
    else:
        tx, ty = hrg * h, wrg * w
    translation_matrix = np.array([[1, 0, tx], [0, 1, ty], [0, 0, 1]])

    transform_matrix = translation_matrix  # no need to do offset
    x = affine_transform(x, transform_matrix, channel_index, fill_mode, cval, order)
    return x


Instruction:
Can you write a function in Python 3 where it
shifts images with the same arguments randomly or non - randomly.

Response:
def shift_multi(
        x, wrg=0.1, hrg=0.1, is_random=False, row_index=0, col_index=1, channel_index=2, fill_mode='nearest', cval=0.,
        order=1
):
    """Shift images with the same arguments, randomly or non-randomly.
    Usually be used for image segmentation which x=[X, Y], X and Y should be matched.

    Parameters
    -----------
    x : list of numpy.array
        List of images with dimension of [n_images, row, col, channel] (default).
    others : args
        See ``tl.prepro.shift``.

    Returns
    -------
    numpy.array
        A list of processed images.

    """
    h, w = x[0].shape[row_index], x[0].shape[col_index]
    if is_random:
        tx = np.random.uniform(-hrg, hrg) * h
        ty = np.random.uniform(-wrg, wrg) * w
    else:
        tx, ty = hrg * h, wrg * w
    translation_matrix = np.array([[1, 0, tx], [0, 1, ty], [0, 0, 1]])

    transform_matrix = translation_matrix  # no need to do offset
    results = []
    for data in x:
        results.append(affine_transform(data, transform_matrix, channel_index, fill_mode, cval, order))
    return np.asarray(results)


Instruction:
Create a Python 3 function for
shearing an image randomly or non - randomly.

Response:
def shear(
        x, intensity=0.1, is_random=False, row_index=0, col_index=1, channel_index=2, fill_mode='nearest', cval=0.,
        order=1
):
    """Shear an image randomly or non-randomly.

    Parameters
    -----------
    x : numpy.array
        An image with dimension of [row, col, channel] (default).
    intensity : float
        Percentage of shear, usually -0.5 ~ 0.5 (is_random==True), 0 ~ 0.5 (is_random==False),
        you can have a quick try by shear(X, 1).
    is_random : boolean
        If True, randomly shear. Default is False.
    row_index col_index and channel_index : int
        Index of row, col and channel, default (0, 1, 2), for theano (1, 2, 0).
    fill_mode : str
        Method to fill missing pixel, default `nearest`, more options `constant`, `reflect` or `wrap`, see and `scipy ndimage affine_transform <https://docs.scipy.org/doc/scipy-0.14.0/reference/generated/scipy.ndimage.interpolation.affine_transform.html>`__
    cval : float
        Value used for points outside the boundaries of the input if mode='constant'. Default is 0.0.
    order : int
        The order of interpolation. The order has to be in the range 0-5. See ``tl.prepro.affine_transform`` and `scipy ndimage affine_transform <https://docs.scipy.org/doc/scipy-0.14.0/reference/generated/scipy.ndimage.interpolation.affine_transform.html>`__

    Returns
    -------
    numpy.array
        A processed image.

    References
    -----------
    - `Affine transformation <https://uk.mathworks.com/discovery/affine-transformation.html>`__

    """
    if is_random:
        shear = np.random.uniform(-intensity, intensity)
    else:
        shear = intensity
    shear_matrix = np.array([[1, -np.sin(shear), 0], [0, np.cos(shear), 0], [0, 0, 1]])

    h, w = x.shape[row_index], x.shape[col_index]
    transform_matrix = transform_matrix_offset_center(shear_matrix, h, w)
    x = affine_transform(x, transform_matrix, channel_index, fill_mode, cval, order)
    return x


Instruction:
Implement a function in Python 3 to
shear an image randomly or non - randomly.

Response:
def shear2(
        x, shear=(0.1, 0.1), is_random=False, row_index=0, col_index=1, channel_index=2, fill_mode='nearest', cval=0.,
        order=1
):
    """Shear an image randomly or non-randomly.

    Parameters
    -----------
    x : numpy.array
        An image with dimension of [row, col, channel] (default).
    shear : tuple of two floats
        Percentage of shear for height and width direction (0, 1).
    is_random : boolean
        If True, randomly shear. Default is False.
    row_index col_index and channel_index : int
        Index of row, col and channel, default (0, 1, 2), for theano (1, 2, 0).
    fill_mode : str
        Method to fill missing pixel, default `nearest`, more options `constant`, `reflect` or `wrap`, see `scipy ndimage affine_transform <https://docs.scipy.org/doc/scipy-0.14.0/reference/generated/scipy.ndimage.interpolation.affine_transform.html>`__
    cval : float
        Value used for points outside the boundaries of the input if mode='constant'. Default is 0.0.
    order : int
        The order of interpolation. The order has to be in the range 0-5. See ``tl.prepro.affine_transform`` and `scipy ndimage affine_transform <https://docs.scipy.org/doc/scipy-0.14.0/reference/generated/scipy.ndimage.interpolation.affine_transform.html>`__

    Returns
    -------
    numpy.array
        A processed image.

    References
    -----------
    - `Affine transformation <https://uk.mathworks.com/discovery/affine-transformation.html>`__

    """
    if len(shear) != 2:
        raise AssertionError(
            "shear should be tuple of 2 floats, or you want to use tl.prepro.shear rather than tl.prepro.shear2 ?"
        )
    if isinstance(shear, tuple):
        shear = list(shear)
    if is_random:
        shear[0] = np.random.uniform(-shear[0], shear[0])
        shear[1] = np.random.uniform(-shear[1], shear[1])

    shear_matrix = np.array([[1, shear[0], 0], \
                            [shear[1], 1, 0], \
                            [0, 0, 1]])

    h, w = x.shape[row_index], x.shape[col_index]
    transform_matrix = transform_matrix_offset_center(shear_matrix, h, w)
    x = affine_transform(x, transform_matrix, channel_index, fill_mode, cval, order)
    return x


Instruction:
Write a Python 3 script for
generating a swirl image.

Response:
def swirl(
        x, center=None, strength=1, radius=100, rotation=0, output_shape=None, order=1, mode='constant', cval=0,
        clip=True, preserve_range=False, is_random=False
):
    """Swirl an image randomly or non-randomly, see `scikit-image swirl API <http://scikit-image.org/docs/dev/api/skimage.transform.html#skimage.transform.swirl>`__
    and `example <http://scikit-image.org/docs/dev/auto_examples/plot_swirl.html>`__.

    Parameters
    -----------
    x : numpy.array
        An image with dimension of [row, col, channel] (default).
    center : tuple or 2 int or None
        Center coordinate of transformation (optional).
    strength : float
        The amount of swirling applied.
    radius : float
        The extent of the swirl in pixels. The effect dies out rapidly beyond radius.
    rotation : float
        Additional rotation applied to the image, usually [0, 360], relates to center.
    output_shape : tuple of 2 int or None
        Shape of the output image generated (height, width). By default the shape of the input image is preserved.
    order : int, optional
        The order of the spline interpolation, default is 1. The order has to be in the range 0-5. See skimage.transform.warp for detail.
    mode : str
        One of `constant` (default), `edge`, `symmetric` `reflect` and `wrap`.
        Points outside the boundaries of the input are filled according to the given mode, with `constant` used as the default. Modes match the behaviour of numpy.pad.
    cval : float
        Used in conjunction with mode `constant`, the value outside the image boundaries.
    clip : boolean
        Whether to clip the output to the range of values of the input image. This is enabled by default, since higher order interpolation may produce values outside the given input range.
    preserve_range : boolean
        Whether to keep the original range of values. Otherwise, the input image is converted according to the conventions of img_as_float.
    is_random : boolean,
        If True, random swirl. Default is False.
            - random center = [(0 ~ x.shape[0]), (0 ~ x.shape[1])]
            - random strength = [0, strength]
            - random radius = [1e-10, radius]
            - random rotation = [-rotation, rotation]

    Returns
    -------
    numpy.array
        A processed image.

    Examples
    ---------
    >>> x --> [row, col, 1] greyscale
    >>> x = tl.prepro.swirl(x, strength=4, radius=100)

    """
    if radius == 0:
        raise AssertionError("Invalid radius value")

    rotation = np.pi / 180 * rotation
    if is_random:
        center_h = int(np.random.uniform(0, x.shape[0]))
        center_w = int(np.random.uniform(0, x.shape[1]))
        center = (center_h, center_w)
        strength = np.random.uniform(0, strength)
        radius = np.random.uniform(1e-10, radius)
        rotation = np.random.uniform(-rotation, rotation)

    max_v = np.max(x)
    if max_v > 1:  # Note: the input of this fn should be [-1, 1], rescale is required.
        x = x / max_v
    swirled = skimage.transform.swirl(
        x, center=center, strength=strength, radius=radius, rotation=rotation, output_shape=output_shape, order=order,
        mode=mode, cval=cval, clip=clip, preserve_range=preserve_range
    )
    if max_v > 1:
        swirled = swirled * max_v
    return swirled


Instruction:
Can you tell what is the following Python 3 function doing
def elastic_transform(x, alpha, sigma, mode="constant", cval=0, is_random=False):
    if is_random is False:
        random_state = np.random.RandomState(None)
    else:
        random_state = np.random.RandomState(int(time.time()))
    #
    is_3d = False
    if len(x.shape) == 3 and x.shape[-1] == 1:
        x = x[:, :, 0]
        is_3d = True
    elif len(x.shape) == 3 and x.shape[-1] != 1:
        raise Exception("Only support greyscale image")

    if len(x.shape) != 2:
        raise AssertionError("input should be grey-scale image")

    shape = x.shape

    dx = gaussian_filter((random_state.rand(*shape) * 2 - 1), sigma, mode=mode, cval=cval) * alpha
    dy = gaussian_filter((random_state.rand(*shape) * 2 - 1), sigma, mode=mode, cval=cval) * alpha

    x_, y_ = np.meshgrid(np.arange(shape[0]), np.arange(shape[1]), indexing='ij')
    indices = np.reshape(x_ + dx, (-1, 1)), np.reshape(y_ + dy, (-1, 1))
    if is_3d:
        return map_coordinates(x, indices, order=1).reshape((shape[0], shape[1], 1))
    else:
        return map_coordinates(x, indices, order=1).reshape(shape)

Response: Elastic transformation for image as described in [ Simard2003 ].


Instruction:
How would you explain what the following Python 3 function does
def zoom(x, zoom_range=(0.9, 1.1), flags=None, border_mode='constant'):
    zoom_matrix = affine_zoom_matrix(zoom_range=zoom_range)
    h, w = x.shape[0], x.shape[1]
    transform_matrix = transform_matrix_offset_center(zoom_matrix, h, w)
    x = affine_transform_cv2(x, transform_matrix, flags=flags, border_mode=border_mode)
    return x

Response: Zooms a single image that height and width are changed together.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def respective_zoom(x, h_range=(0.9, 1.1), w_range=(0.9, 1.1), flags=None, border_mode='constant'):
    zoom_matrix = affine_respective_zoom_matrix(h_range=h_range, w_range=w_range)
    h, w = x.shape[0], x.shape[1]
    transform_matrix = transform_matrix_offset_center(zoom_matrix, h, w)
    x = affine_transform_cv2(
        x, transform_matrix, flags=flags, border_mode=border_mode
    )  #affine_transform(x, transform_matrix, channel_index, fill_mode, cval, order)
    return x

Response: Respects the height and width of a single image.


Instruction:
Here you have a function in Python 3, explain what it does
def zoom_multi(x, zoom_range=(0.9, 1.1), flags=None, border_mode='constant'):

    zoom_matrix = affine_zoom_matrix(zoom_range=zoom_range)
    results = []
    for img in x:
        h, w = x.shape[0], x.shape[1]
        transform_matrix = transform_matrix_offset_center(zoom_matrix, h, w)
        results.append(affine_transform_cv2(x, transform_matrix, flags=flags, border_mode=border_mode))
    return result

Response: Zooms in and out of images with the same arguments randomly or non - randomly.


Instruction:
Explain what the following Python 3 code does
def brightness(x, gamma=1, gain=1, is_random=False):
    if is_random:
        gamma = np.random.uniform(1 - gamma, 1 + gamma)
    x = exposure.adjust_gamma(x, gamma, gain)
    return x

Response: Change the brightness of a single image randomly or non - randomly.


Instruction:
Here you have a function in Python 3, explain what it does
def brightness_multi(x, gamma=1, gain=1, is_random=False):
    if is_random:
        gamma = np.random.uniform(1 - gamma, 1 + gamma)

    results = []
    for data in x:
        results.append(exposure.adjust_gamma(data, gamma, gain))
    return np.asarray(results)

Response: Change the brightness of multiply images randomly or non - randomly.


Instruction:
Can you implement a function in Python 3 that
performs illumination augmentation for a single image.

Response:
def illumination(x, gamma=1., contrast=1., saturation=1., is_random=False):
    """Perform illumination augmentation for a single image, randomly or non-randomly.

    Parameters
    -----------
    x : numpy.array
        An image with dimension of [row, col, channel] (default).
    gamma : float
        Change brightness (the same with ``tl.prepro.brightness``)
            - if is_random=False, one float number, small than one means brighter, greater than one means darker.
            - if is_random=True, tuple of two float numbers, (min, max).
    contrast : float
        Change contrast.
            - if is_random=False, one float number, small than one means blur.
            - if is_random=True, tuple of two float numbers, (min, max).
    saturation : float
        Change saturation.
            - if is_random=False, one float number, small than one means unsaturation.
            - if is_random=True, tuple of two float numbers, (min, max).
    is_random : boolean
        If True, randomly change illumination. Default is False.

    Returns
    -------
    numpy.array
        A processed image.

    Examples
    ---------
    Random

    >>> x = tl.prepro.illumination(x, gamma=(0.5, 5.0), contrast=(0.3, 1.0), saturation=(0.7, 1.0), is_random=True)

    Non-random

    >>> x = tl.prepro.illumination(x, 0.5, 0.6, 0.8, is_random=False)

    """
    if is_random:
        if not (len(gamma) == len(contrast) == len(saturation) == 2):
            raise AssertionError("if is_random = True, the arguments are (min, max)")

        ## random change brightness  # small --> brighter
        illum_settings = np.random.randint(0, 3)  # 0-brighter, 1-darker, 2 keep normal

        if illum_settings == 0:  # brighter
            gamma = np.random.uniform(gamma[0], 1.0)  # (.5, 1.0)
        elif illum_settings == 1:  # darker
            gamma = np.random.uniform(1.0, gamma[1])  # (1.0, 5.0)
        else:
            gamma = 1
        im_ = brightness(x, gamma=gamma, gain=1, is_random=False)

        # tl.logging.info("using contrast and saturation")
        image = PIL.Image.fromarray(im_)  # array -> PIL
        contrast_adjust = PIL.ImageEnhance.Contrast(image)
        image = contrast_adjust.enhance(np.random.uniform(contrast[0], contrast[1]))  #0.3,0.9))

        saturation_adjust = PIL.ImageEnhance.Color(image)
        image = saturation_adjust.enhance(np.random.uniform(saturation[0], saturation[1]))  # (0.7,1.0))
        im_ = np.array(image)  # PIL -> array
    else:
        im_ = brightness(x, gamma=gamma, gain=1, is_random=False)
        image = PIL.Image.fromarray(im_)  # array -> PIL
        contrast_adjust = PIL.ImageEnhance.Contrast(image)
        image = contrast_adjust.enhance(contrast)

        saturation_adjust = PIL.ImageEnhance.Color(image)
        image = saturation_adjust.enhance(saturation)
        im_ = np.array(image)  # PIL -> array
    return np.asarray(im_)


Instruction:
Write a Python 3 script for
converting RGB image to HSV image.

Response:
def rgb_to_hsv(rgb):
    """Input RGB image [0~255] return HSV image [0~1].

    Parameters
    ------------
    rgb : numpy.array
        An image with values between 0 and 255.

    Returns
    -------
    numpy.array
        A processed image.

    """
    # Translated from source of colorsys.rgb_to_hsv
    # r,g,b should be a numpy arrays with values between 0 and 255
    # rgb_to_hsv returns an array of floats between 0.0 and 1.0.
    rgb = rgb.astype('float')
    hsv = np.zeros_like(rgb)
    # in case an RGBA array was passed, just copy the A channel
    hsv[..., 3:] = rgb[..., 3:]
    r, g, b = rgb[..., 0], rgb[..., 1], rgb[..., 2]
    maxc = np.max(rgb[..., :3], axis=-1)
    minc = np.min(rgb[..., :3], axis=-1)
    hsv[..., 2] = maxc
    mask = maxc != minc
    hsv[mask, 1] = (maxc - minc)[mask] / maxc[mask]
    rc = np.zeros_like(r)
    gc = np.zeros_like(g)
    bc = np.zeros_like(b)
    rc[mask] = (maxc - r)[mask] / (maxc - minc)[mask]
    gc[mask] = (maxc - g)[mask] / (maxc - minc)[mask]
    bc[mask] = (maxc - b)[mask] / (maxc - minc)[mask]
    hsv[..., 0] = np.select([r == maxc, g == maxc], [bc - gc, 2.0 + rc - bc], default=4.0 + gc - rc)
    hsv[..., 0] = (hsv[..., 0] / 6.0) % 1.0
    return hsv


Instruction:
Can you generate a brief explanation for the following Python 3 code
def hsv_to_rgb(hsv):
    # Translated from source of colorsys.hsv_to_rgb
    # h,s should be a numpy arrays with values between 0.0 and 1.0
    # v should be a numpy array with values between 0.0 and 255.0
    # hsv_to_rgb returns an array of uints between 0 and 255.
    rgb = np.empty_like(hsv)
    rgb[..., 3:] = hsv[..., 3:]
    h, s, v = hsv[..., 0], hsv[..., 1], hsv[..., 2]
    i = (h * 6.0).astype('uint8')
    f = (h * 6.0) - i
    p = v * (1.0 - s)
    q = v * (1.0 - s * f)
    t = v * (1.0 - s * (1.0 - f))
    i = i % 6
    conditions = [s == 0.0, i == 1, i == 2, i == 3, i == 4, i == 5]
    rgb[..., 0] = np.select(conditions, [v, q, p, p, t, v], default=v)
    rgb[..., 1] = np.select(conditions, [v, v, v, q, p, p], default=t)
    rgb[..., 2] = np.select(conditions, [v, p, t, v, v, q], default=p)
    return rgb.astype('uint8')

Response: This function converts an HSV image to RGB image.


Instruction:
Create a Python 3 function for
adjusting the hue of an RGB image.

Response:
def adjust_hue(im, hout=0.66, is_offset=True, is_clip=True, is_random=False):
    """Adjust hue of an RGB image.

    This is a convenience method that converts an RGB image to float representation, converts it to HSV, add an offset to the hue channel, converts back to RGB and then back to the original data type.
    For TF, see `tf.image.adjust_hue <https://www.tensorflow.org/api_docs/python/tf/image/adjust_hue>`__.and `tf.image.random_hue <https://www.tensorflow.org/api_docs/python/tf/image/random_hue>`__.

    Parameters
    -----------
    im : numpy.array
        An image with values between 0 and 255.
    hout : float
        The scale value for adjusting hue.
            - If is_offset is False, set all hue values to this value. 0 is red; 0.33 is green; 0.66 is blue.
            - If is_offset is True, add this value as the offset to the hue channel.
    is_offset : boolean
        Whether `hout` is added on HSV as offset or not. Default is True.
    is_clip : boolean
        If HSV value smaller than 0, set to 0. Default is True.
    is_random : boolean
        If True, randomly change hue. Default is False.

    Returns
    -------
    numpy.array
        A processed image.

    Examples
    ---------
    Random, add a random value between -0.2 and 0.2 as the offset to every hue values.

    >>> im_hue = tl.prepro.adjust_hue(image, hout=0.2, is_offset=True, is_random=False)

    Non-random, make all hue to green.

    >>> im_green = tl.prepro.adjust_hue(image, hout=0.66, is_offset=False, is_random=False)

    References
    -----------
    - `tf.image.random_hue <https://www.tensorflow.org/api_docs/python/tf/image/random_hue>`__.
    - `tf.image.adjust_hue <https://www.tensorflow.org/api_docs/python/tf/image/adjust_hue>`__.
    - `StackOverflow: Changing image hue with python PIL <https://stackoverflow.com/questions/7274221/changing-image-hue-with-python-pil>`__.

    """
    hsv = rgb_to_hsv(im)
    if is_random:
        hout = np.random.uniform(-hout, hout)

    if is_offset:
        hsv[..., 0] += hout
    else:
        hsv[..., 0] = hout

    if is_clip:
        hsv[..., 0] = np.clip(hsv[..., 0], 0, np.inf)  # Hao : can remove green dots

    rgb = hsv_to_rgb(hsv)
    return rgb


Instruction:
Given the following Python 3 function, write the documentation
def imresize(x, size=None, interp='bicubic', mode=None):
    if size is None:
        size = [100, 100]

    if x.shape[-1] == 1:
        # greyscale
        x = scipy.misc.imresize(x[:, :, 0], size, interp=interp, mode=mode)
        return x[:, :, np.newaxis]
    else:
        # rgb, bgr, rgba
        return scipy.misc.imresize(x, size, interp=interp, mode=mode)

Response: Resizes an image by given output size and interpolation method.


Instruction:
Can you tell what is the following Python 3 function doing
def pixel_value_scale(im, val=0.9, clip=None, is_random=False):

    clip = clip if clip is not None else (-np.inf, np.inf)

    if is_random:
        scale = 1 + np.random.uniform(-val, val)
        im = im * scale
    else:
        im = im * val

    if len(clip) == 2:
        im = np.clip(im, clip[0], clip[1])
    else:
        raise Exception("clip : tuple of 2 numbers")

    return im

Response: Scales each value in the pixels of the image.


Instruction:
How would you explain what the following Python 3 function does
def samplewise_norm(
        x, rescale=None, samplewise_center=False, samplewise_std_normalization=False, channel_index=2, epsilon=1e-7
):
    """Normalize an image by rescale, samplewise centering and samplewise centering in order.

    Parameters
    -----------
    x : numpy.array
        An image with dimension of [row, col, channel] (default).
    rescale : float
        Rescaling factor. If None or 0, no rescaling is applied, otherwise we multiply the data by the value provided (before applying any other transformation)
    samplewise_center : boolean
        If True, set each sample mean to 0.
    samplewise_std_normalization : boolean
        If True, divide each input by its std.
    epsilon : float
        A small position value for dividing standard deviation.

    Returns
    -------
    numpy.array
        A processed image.

    Examples
    --------
    >>> x = samplewise_norm(x, samplewise_center=True, samplewise_std_normalization=True)
    >>> print(x.shape, np.mean(x), np.std(x))
    (160, 176, 1), 0.0, 1.0

    Notes
    ------
    When samplewise_center and samplewise_std_normalization are True.
    - For greyscale image, every pixels are subtracted and divided by the mean and std of whole image.
    - For RGB image, every pixels are subtracted and divided by the mean and std of this pixel i.e. the mean and std of a pixel is 0 and 1.

    """
    if rescale:
        x *= rescale

    if x.shape[channel_index] == 1:
        # greyscale
        if samplewise_center:
            x = x - np.mean(x)
        if samplewise_std_normalization:
            x = x / np.std(x)
        return x
    elif x.shape[channel_index] == 3:
        # rgb
        if samplewise_center:
            x = x - np.mean(x, axis=channel_index, keepdims=True)
        if samplewise_std_normalization:
            x = x / (np.std(x, axis=channel_index, keepdims=True) + epsilon)
        return x
    else:
        raise Exception("Unsupported channels %d" % x.shape[channel_index])

Response: This function normalizes an image by sampling the image mean and standard deviation of each input image.


Instruction:
Given the following Python 3 function, write the documentation
def featurewise_norm(x, mean=None, std=None, epsilon=1e-7):
    if mean:
        x = x - mean
    if std:
        x = x / (std + epsilon)
    return x

Response:
Normalize every pixel by the same given mean and std which are usually
    compute from all examples.