Delete training

training/__init__.py

@@ -1,7 +0,0 @@
-# Copyright (c) 2019, NVIDIA Corporation. All rights reserved.
-#
-# This work is made available under the Nvidia Source Code License-NC.
-# To view a copy of this license, visit
-# https://nvlabs.github.io/stylegan2/license.html
-
-# empty

training/dataset.py

@@ -1,199 +0,0 @@
-# Copyright (c) 2019, NVIDIA Corporation. All rights reserved.
-#
-# This work is made available under the Nvidia Source Code License-NC.
-# To view a copy of this license, visit
-# https://nvlabs.github.io/stylegan2/license.html
-
-"""Multi-resolution input data pipeline."""
-
-import os
-import glob
-import numpy as np
-import tensorflow as tf
-import dnnlib
-import dnnlib.tflib as tflib
-
-#----------------------------------------------------------------------------
-# Dataset class that loads data from tfrecords files.
-
-class TFRecordDataset:
-    def __init__(self,
-        tfrecord_dir,               # Directory containing a collection of tfrecords files.
-        resolution      = None,     # Dataset resolution, None = autodetect.
-        label_file      = None,     # Relative path of the labels file, None = autodetect.
-        max_label_size  = 0,        # 0 = no labels, 'full' = full labels, <int> = N first label components.
-        max_images      = None,     # Maximum number of images to use, None = use all images.
-        repeat          = True,     # Repeat dataset indefinitely?
-        shuffle_mb      = 4096,     # Shuffle data within specified window (megabytes), 0 = disable shuffling.
-        prefetch_mb     = 2048,     # Amount of data to prefetch (megabytes), 0 = disable prefetching.
-        buffer_mb       = 256,      # Read buffer size (megabytes).
-        num_threads     = 2):       # Number of concurrent threads.
-
-        self.tfrecord_dir       = tfrecord_dir
-        self.resolution         = None
-        self.resolution_log2    = None
-        self.shape              = []        # [channels, height, width]
-        self.dtype              = 'uint8'
-        self.dynamic_range      = [0, 255]
-        self.label_file         = label_file
-        self.label_size         = None      # components
-        self.label_dtype        = None
-        self._np_labels         = None
-        self._tf_minibatch_in   = None
-        self._tf_labels_var     = None
-        self._tf_labels_dataset = None
-        self._tf_datasets       = dict()
-        self._tf_iterator       = None
-        self._tf_init_ops       = dict()
-        self._tf_minibatch_np   = None
-        self._cur_minibatch     = -1
-        self._cur_lod           = -1
-
-        # List tfrecords files and inspect their shapes.
-        assert os.path.isdir(self.tfrecord_dir)
-        tfr_files = sorted(glob.glob(os.path.join(self.tfrecord_dir, '*.tfrecords')))
-        assert len(tfr_files) >= 1
-        tfr_shapes = []
-        for tfr_file in tfr_files:
-            tfr_opt = tf.python_io.TFRecordOptions(tf.python_io.TFRecordCompressionType.NONE)
-            for record in tf.python_io.tf_record_iterator(tfr_file, tfr_opt):
-                tfr_shapes.append(self.parse_tfrecord_np(record).shape)
-                break
-
-        # Autodetect label filename.
-        if self.label_file is None:
-            guess = sorted(glob.glob(os.path.join(self.tfrecord_dir, '*.labels')))
-            if len(guess):
-                self.label_file = guess[0]
-        elif not os.path.isfile(self.label_file):
-            guess = os.path.join(self.tfrecord_dir, self.label_file)
-            if os.path.isfile(guess):
-                self.label_file = guess
-
-        # Determine shape and resolution.
-        max_shape = max(tfr_shapes, key=np.prod)
-        self.resolution = resolution if resolution is not None else max_shape[1]
-        self.resolution_log2 = int(np.log2(self.resolution))
-        self.shape = [max_shape[0], self.resolution, self.resolution]
-        tfr_lods = [self.resolution_log2 - int(np.log2(shape[1])) for shape in tfr_shapes]
-        assert all(shape[0] == max_shape[0] for shape in tfr_shapes)
-        assert all(shape[1] == shape[2] for shape in tfr_shapes)
-        assert all(shape[1] == self.resolution // (2**lod) for shape, lod in zip(tfr_shapes, tfr_lods))
-        assert all(lod in tfr_lods for lod in range(self.resolution_log2 - 1))
-
-        # Load labels.
-        assert max_label_size == 'full' or max_label_size >= 0
-        self._np_labels = np.zeros([1<<30, 0], dtype=np.float32)
-        if self.label_file is not None and max_label_size != 0:
-            self._np_labels = np.load(self.label_file)
-            assert self._np_labels.ndim == 2
-        if max_label_size != 'full' and self._np_labels.shape[1] > max_label_size:
-            self._np_labels = self._np_labels[:, :max_label_size]
-        if max_images is not None and self._np_labels.shape[0] > max_images:
-            self._np_labels = self._np_labels[:max_images]
-        self.label_size = self._np_labels.shape[1]
-        self.label_dtype = self._np_labels.dtype.name
-
-        # Build TF expressions.
-        with tf.name_scope('Dataset'), tf.device('/cpu:0'):
-            self._tf_minibatch_in = tf.placeholder(tf.int64, name='minibatch_in', shape=[])
-            self._tf_labels_var = tflib.create_var_with_large_initial_value(self._np_labels, name='labels_var')
-            self._tf_labels_dataset = tf.data.Dataset.from_tensor_slices(self._tf_labels_var)
-            for tfr_file, tfr_shape, tfr_lod in zip(tfr_files, tfr_shapes, tfr_lods):
-                if tfr_lod < 0:
-                    continue
-                dset = tf.data.TFRecordDataset(tfr_file, compression_type='', buffer_size=buffer_mb<<20)
-                if max_images is not None:
-                    dset = dset.take(max_images)
-                dset = dset.map(self.parse_tfrecord_tf, num_parallel_calls=num_threads)
-                dset = tf.data.Dataset.zip((dset, self._tf_labels_dataset))
-                bytes_per_item = np.prod(tfr_shape) * np.dtype(self.dtype).itemsize
-                if shuffle_mb > 0:
-                    dset = dset.shuffle(((shuffle_mb << 20) - 1) // bytes_per_item + 1)
-                if repeat:
-                    dset = dset.repeat()
-                if prefetch_mb > 0:
-                    dset = dset.prefetch(((prefetch_mb << 20) - 1) // bytes_per_item + 1)
-                dset = dset.batch(self._tf_minibatch_in)
-                self._tf_datasets[tfr_lod] = dset
-            self._tf_iterator = tf.data.Iterator.from_structure(self._tf_datasets[0].output_types, self._tf_datasets[0].output_shapes)
-            self._tf_init_ops = {lod: self._tf_iterator.make_initializer(dset) for lod, dset in self._tf_datasets.items()}
-
-    def close(self):
-        pass
-
-    # Use the given minibatch size and level-of-detail for the data returned by get_minibatch_tf().
-    def configure(self, minibatch_size, lod=0):
-        lod = int(np.floor(lod))
-        assert minibatch_size >= 1 and lod in self._tf_datasets
-        if self._cur_minibatch != minibatch_size or self._cur_lod != lod:
-            self._tf_init_ops[lod].run({self._tf_minibatch_in: minibatch_size})
-            self._cur_minibatch = minibatch_size
-            self._cur_lod = lod
-
-    # Get next minibatch as TensorFlow expressions.
-    def get_minibatch_tf(self): # => images, labels
-        return self._tf_iterator.get_next()
-
-    # Get next minibatch as NumPy arrays.
-    def get_minibatch_np(self, minibatch_size, lod=0): # => images, labels
-        self.configure(minibatch_size, lod)
-        with tf.name_scope('Dataset'):
-            if self._tf_minibatch_np is None:
-                self._tf_minibatch_np = self.get_minibatch_tf()
-            return tflib.run(self._tf_minibatch_np)
-
-    # Get random labels as TensorFlow expression.
-    def get_random_labels_tf(self, minibatch_size): # => labels
-        with tf.name_scope('Dataset'):
-            if self.label_size > 0:
-                with tf.device('/cpu:0'):
-                    return tf.gather(self._tf_labels_var, tf.random_uniform([minibatch_size], 0, self._np_labels.shape[0], dtype=tf.int32))
-            return tf.zeros([minibatch_size, 0], self.label_dtype)
-
-    # Get random labels as NumPy array.
-    def get_random_labels_np(self, minibatch_size): # => labels
-        if self.label_size > 0:
-            return self._np_labels[np.random.randint(self._np_labels.shape[0], size=[minibatch_size])]
-        return np.zeros([minibatch_size, 0], self.label_dtype)
-
-    # Parse individual image from a tfrecords file into TensorFlow expression.
-    @staticmethod
-    def parse_tfrecord_tf(record):
-        features = tf.parse_single_example(record, features={
-            'shape': tf.FixedLenFeature([3], tf.int64),
-            'data': tf.FixedLenFeature([], tf.string)})
-        data = tf.decode_raw(features['data'], tf.uint8)
-        return tf.reshape(data, features['shape'])
-
-    # Parse individual image from a tfrecords file into NumPy array.
-    @staticmethod
-    def parse_tfrecord_np(record):
-        ex = tf.train.Example()
-        ex.ParseFromString(record)
-        shape = ex.features.feature['shape'].int64_list.value # pylint: disable=no-member
-        data = ex.features.feature['data'].bytes_list.value[0] # pylint: disable=no-member
-        return np.fromstring(data, np.uint8).reshape(shape)
-
-#----------------------------------------------------------------------------
-# Helper func for constructing a dataset object using the given options.
-
-def load_dataset(class_name=None, data_dir=None, verbose=False, **kwargs):
-    kwargs = dict(kwargs)
-    if 'tfrecord_dir' in kwargs:
-        if class_name is None:
-            class_name = __name__ + '.TFRecordDataset'
-        if data_dir is not None:
-            kwargs['tfrecord_dir'] = os.path.join(data_dir, kwargs['tfrecord_dir'])
-
-    assert class_name is not None
-    if verbose:
-        print('Streaming data using %s...' % class_name)
-    dataset = dnnlib.util.get_obj_by_name(class_name)(**kwargs)
-    if verbose:
-        print('Dataset shape =', np.int32(dataset.shape).tolist())
-        print('Dynamic range =', dataset.dynamic_range)
-        print('Label size    =', dataset.label_size)
-    return dataset
-
-#----------------------------------------------------------------------------

training/loss.py

@@ -1,197 +0,0 @@
-# Copyright (c) 2019, NVIDIA Corporation. All rights reserved.
-#
-# This work is made available under the Nvidia Source Code License-NC.
-# To view a copy of this license, visit
-# https://nvlabs.github.io/stylegan2/license.html
-
-"""Loss functions."""
-
-import numpy as np
-import tensorflow as tf
-import dnnlib.tflib as tflib
-from dnnlib.tflib.autosummary import autosummary
-
-#----------------------------------------------------------------------------
-# Logistic loss from the paper
-# "Generative Adversarial Nets", Goodfellow et al. 2014
-
-def G_logistic(G, D, opt, training_set, minibatch_size):
-    _ = opt
-    latents = tf.random_normal([minibatch_size] + G.input_shapes[0][1:])
-    labels = training_set.get_random_labels_tf(minibatch_size)
-    fake_images_out = G.get_output_for(latents, labels, is_training=True)
-    fake_scores_out = D.get_output_for(fake_images_out, labels, is_training=True)
-    loss = -tf.nn.softplus(fake_scores_out) # log(1-sigmoid(fake_scores_out)) # pylint: disable=invalid-unary-operand-type
-    return loss, None
-
-def G_logistic_ns(G, D, opt, training_set, minibatch_size):
-    _ = opt
-    latents = tf.random_normal([minibatch_size] + G.input_shapes[0][1:])
-    labels = training_set.get_random_labels_tf(minibatch_size)
-    fake_images_out = G.get_output_for(latents, labels, is_training=True)
-    fake_scores_out = D.get_output_for(fake_images_out, labels, is_training=True)
-    loss = tf.nn.softplus(-fake_scores_out) # -log(sigmoid(fake_scores_out))
-    return loss, None
-
-def D_logistic(G, D, opt, training_set, minibatch_size, reals, labels):
-    _ = opt, training_set
-    latents = tf.random_normal([minibatch_size] + G.input_shapes[0][1:])
-    fake_images_out = G.get_output_for(latents, labels, is_training=True)
-    real_scores_out = D.get_output_for(reals, labels, is_training=True)
-    fake_scores_out = D.get_output_for(fake_images_out, labels, is_training=True)
-    real_scores_out = autosummary('Loss/scores/real', real_scores_out)
-    fake_scores_out = autosummary('Loss/scores/fake', fake_scores_out)
-    loss = tf.nn.softplus(fake_scores_out) # -log(1-sigmoid(fake_scores_out))
-    loss += tf.nn.softplus(-real_scores_out) # -log(sigmoid(real_scores_out)) # pylint: disable=invalid-unary-operand-type
-    return loss, None
-
-#----------------------------------------------------------------------------
-# R1 and R2 regularizers from the paper
-# "Which Training Methods for GANs do actually Converge?", Mescheder et al. 2018
-
-def D_logistic_r1(G, D, opt, training_set, minibatch_size, reals, labels, gamma=10.0):
-    _ = opt, training_set
-    latents = tf.random_normal([minibatch_size] + G.input_shapes[0][1:])
-    fake_images_out = G.get_output_for(latents, labels, is_training=True)
-    real_scores_out = D.get_output_for(reals, labels, is_training=True)
-    fake_scores_out = D.get_output_for(fake_images_out, labels, is_training=True)
-    real_scores_out = autosummary('Loss/scores/real', real_scores_out)
-    fake_scores_out = autosummary('Loss/scores/fake', fake_scores_out)
-    loss = tf.nn.softplus(fake_scores_out) # -log(1-sigmoid(fake_scores_out))
-    loss += tf.nn.softplus(-real_scores_out) # -log(sigmoid(real_scores_out)) # pylint: disable=invalid-unary-operand-type
-
-    with tf.name_scope('GradientPenalty'):
-        real_grads = tf.gradients(tf.reduce_sum(real_scores_out), [reals])[0]
-        gradient_penalty = tf.reduce_sum(tf.square(real_grads), axis=[1,2,3])
-        gradient_penalty = autosummary('Loss/gradient_penalty', gradient_penalty)
-        reg = gradient_penalty * (gamma * 0.5)
-    return loss, reg
-
-def D_logistic_r2(G, D, opt, training_set, minibatch_size, reals, labels, gamma=10.0):
-    _ = opt, training_set
-    latents = tf.random_normal([minibatch_size] + G.input_shapes[0][1:])
-    fake_images_out = G.get_output_for(latents, labels, is_training=True)
-    real_scores_out = D.get_output_for(reals, labels, is_training=True)
-    fake_scores_out = D.get_output_for(fake_images_out, labels, is_training=True)
-    real_scores_out = autosummary('Loss/scores/real', real_scores_out)
-    fake_scores_out = autosummary('Loss/scores/fake', fake_scores_out)
-    loss = tf.nn.softplus(fake_scores_out) # -log(1-sigmoid(fake_scores_out))
-    loss += tf.nn.softplus(-real_scores_out) # -log(sigmoid(real_scores_out)) # pylint: disable=invalid-unary-operand-type
-
-    with tf.name_scope('GradientPenalty'):
-        fake_grads = tf.gradients(tf.reduce_sum(fake_scores_out), [fake_images_out])[0]
-        gradient_penalty = tf.reduce_sum(tf.square(fake_grads), axis=[1,2,3])
-        gradient_penalty = autosummary('Loss/gradient_penalty', gradient_penalty)
-        reg = gradient_penalty * (gamma * 0.5)
-    return loss, reg
-
-#----------------------------------------------------------------------------
-# WGAN loss from the paper
-# "Wasserstein Generative Adversarial Networks", Arjovsky et al. 2017
-
-def G_wgan(G, D, opt, training_set, minibatch_size):
-    _ = opt
-    latents = tf.random_normal([minibatch_size] + G.input_shapes[0][1:])
-    labels = training_set.get_random_labels_tf(minibatch_size)
-    fake_images_out = G.get_output_for(latents, labels, is_training=True)
-    fake_scores_out = D.get_output_for(fake_images_out, labels, is_training=True)
-    loss = -fake_scores_out
-    return loss, None
-
-def D_wgan(G, D, opt, training_set, minibatch_size, reals, labels, wgan_epsilon=0.001):
-    _ = opt, training_set
-    latents = tf.random_normal([minibatch_size] + G.input_shapes[0][1:])
-    fake_images_out = G.get_output_for(latents, labels, is_training=True)
-    real_scores_out = D.get_output_for(reals, labels, is_training=True)
-    fake_scores_out = D.get_output_for(fake_images_out, labels, is_training=True)
-    real_scores_out = autosummary('Loss/scores/real', real_scores_out)
-    fake_scores_out = autosummary('Loss/scores/fake', fake_scores_out)
-    loss = fake_scores_out - real_scores_out
-    with tf.name_scope('EpsilonPenalty'):
-        epsilon_penalty = autosummary('Loss/epsilon_penalty', tf.square(real_scores_out))
-        loss += epsilon_penalty * wgan_epsilon
-    return loss, None
-
-#----------------------------------------------------------------------------
-# WGAN-GP loss from the paper
-# "Improved Training of Wasserstein GANs", Gulrajani et al. 2017
-
-def D_wgan_gp(G, D, opt, training_set, minibatch_size, reals, labels, wgan_lambda=10.0, wgan_epsilon=0.001, wgan_target=1.0):
-    _ = opt, training_set
-    latents = tf.random_normal([minibatch_size] + G.input_shapes[0][1:])
-    fake_images_out = G.get_output_for(latents, labels, is_training=True)
-    real_scores_out = D.get_output_for(reals, labels, is_training=True)
-    fake_scores_out = D.get_output_for(fake_images_out, labels, is_training=True)
-    real_scores_out = autosummary('Loss/scores/real', real_scores_out)
-    fake_scores_out = autosummary('Loss/scores/fake', fake_scores_out)
-    loss = fake_scores_out - real_scores_out
-    with tf.name_scope('EpsilonPenalty'):
-        epsilon_penalty = autosummary('Loss/epsilon_penalty', tf.square(real_scores_out))
-    loss += epsilon_penalty * wgan_epsilon
-
-    with tf.name_scope('GradientPenalty'):
-        mixing_factors = tf.random_uniform([minibatch_size, 1, 1, 1], 0.0, 1.0, dtype=fake_images_out.dtype)
-        mixed_images_out = tflib.lerp(tf.cast(reals, fake_images_out.dtype), fake_images_out, mixing_factors)
-        mixed_scores_out = D.get_output_for(mixed_images_out, labels, is_training=True)
-        mixed_scores_out = autosummary('Loss/scores/mixed', mixed_scores_out)
-        mixed_grads = tf.gradients(tf.reduce_sum(mixed_scores_out), [mixed_images_out])[0]
-        mixed_norms = tf.sqrt(tf.reduce_sum(tf.square(mixed_grads), axis=[1,2,3]))
-        mixed_norms = autosummary('Loss/mixed_norms', mixed_norms)
-        gradient_penalty = tf.square(mixed_norms - wgan_target)
-        reg = gradient_penalty * (wgan_lambda / (wgan_target**2))
-    return loss, reg
-
-#----------------------------------------------------------------------------
-# Non-saturating logistic loss with path length regularizer from the paper
-# "Analyzing and Improving the Image Quality of StyleGAN", Karras et al. 2019
-
-def G_logistic_ns_pathreg(G, D, opt, training_set, minibatch_size, pl_minibatch_shrink=2, pl_decay=0.01, pl_weight=2.0):
-    _ = opt
-    latents = tf.random_normal([minibatch_size] + G.input_shapes[0][1:])
-    labels = training_set.get_random_labels_tf(minibatch_size)
-    fake_images_out, fake_dlatents_out = G.get_output_for(latents, labels, is_training=True, return_dlatents=True)
-    fake_scores_out = D.get_output_for(fake_images_out, labels, is_training=True)
-    loss = tf.nn.softplus(-fake_scores_out) # -log(sigmoid(fake_scores_out))
-
-    # Path length regularization.
-    with tf.name_scope('PathReg'):
-
-        # Evaluate the regularization term using a smaller minibatch to conserve memory.
-        if pl_minibatch_shrink > 1:
-            pl_minibatch = minibatch_size // pl_minibatch_shrink
-            pl_latents = tf.random_normal([pl_minibatch] + G.input_shapes[0][1:])
-            pl_labels = training_set.get_random_labels_tf(pl_minibatch)
-            fake_images_out, fake_dlatents_out = G.get_output_for(pl_latents, pl_labels, is_training=True, return_dlatents=True)
-
-        # Compute |J*y|.
-        pl_noise = tf.random_normal(tf.shape(fake_images_out)) / np.sqrt(np.prod(G.output_shape[2:]))
-        pl_grads = tf.gradients(tf.reduce_sum(fake_images_out * pl_noise), [fake_dlatents_out])[0]
-        pl_lengths = tf.sqrt(tf.reduce_mean(tf.reduce_sum(tf.square(pl_grads), axis=2), axis=1))
-        pl_lengths = autosummary('Loss/pl_lengths', pl_lengths)
-
-        # Track exponential moving average of |J*y|.
-        with tf.control_dependencies(None):
-            pl_mean_var = tf.Variable(name='pl_mean', trainable=False, initial_value=0.0, dtype=tf.float32)
-        pl_mean = pl_mean_var + pl_decay * (tf.reduce_mean(pl_lengths) - pl_mean_var)
-        pl_update = tf.assign(pl_mean_var, pl_mean)
-
-        # Calculate (|J*y|-a)^2.
-        with tf.control_dependencies([pl_update]):
-            pl_penalty = tf.square(pl_lengths - pl_mean)
-            pl_penalty = autosummary('Loss/pl_penalty', pl_penalty)
-
-        # Apply weight.
-        #
-        # Note: The division in pl_noise decreases the weight by num_pixels, and the reduce_mean
-        # in pl_lengths decreases it by num_affine_layers. The effective weight then becomes:
-        #
-        # gamma_pl = pl_weight / num_pixels / num_affine_layers
-        # = 2 / (r^2) / (log2(r) * 2 - 2)
-        # = 1 / (r^2 * (log2(r) - 1))
-        # = ln(2) / (r^2 * (ln(r) - ln(2))
-        #
-        reg = pl_penalty * pl_weight
-
-    return loss, reg
-
-#----------------------------------------------------------------------------

training/misc.py

@@ -1,145 +0,0 @@
-# Copyright (c) 2019, NVIDIA Corporation. All rights reserved.
-#
-# This work is made available under the Nvidia Source Code License-NC.
-# To view a copy of this license, visit
-# https://nvlabs.github.io/stylegan2/license.html
-
-"""Miscellaneous utility functions."""
-
-import os
-import pickle
-import numpy as np
-import PIL.Image
-import PIL.ImageFont
-import dnnlib
-
-#----------------------------------------------------------------------------
-# Convenience wrappers for pickle that are able to load data produced by
-# older versions of the code, and from external URLs.
-
-def open_file_or_url(file_or_url):
-    if dnnlib.util.is_url(file_or_url):
-        return dnnlib.util.open_url(file_or_url, cache_dir='models')
-    return open(file_or_url, 'rb')
-
-def load_pkl(file_or_url):
-    with open_file_or_url(file_or_url) as file:
-        return pickle.load(file, encoding='latin1')
-
-def save_pkl(obj, filename):
-    with open(filename, 'wb') as file:
-        pickle.dump(obj, file, protocol=pickle.HIGHEST_PROTOCOL)
-
-#----------------------------------------------------------------------------
-# Image utils.
-
-def adjust_dynamic_range(data, drange_in, drange_out):
-    if drange_in != drange_out:
-        scale = (np.float32(drange_out[1]) - np.float32(drange_out[0])) / (np.float32(drange_in[1]) - np.float32(drange_in[0]))
-        bias = (np.float32(drange_out[0]) - np.float32(drange_in[0]) * scale)
-        data = data * scale + bias
-    return data
-
-def create_image_grid(images, grid_size=None):
-    assert images.ndim == 3 or images.ndim == 4
-    num, img_w, img_h = images.shape[0], images.shape[-1], images.shape[-2]
-
-    if grid_size is not None:
-        grid_w, grid_h = tuple(grid_size)
-    else:
-        grid_w = max(int(np.ceil(np.sqrt(num))), 1)
-        grid_h = max((num - 1) // grid_w + 1, 1)
-
-    grid = np.zeros(list(images.shape[1:-2]) + [grid_h * img_h, grid_w * img_w], dtype=images.dtype)
-    for idx in range(num):
-        x = (idx % grid_w) * img_w
-        y = (idx // grid_w) * img_h
-        grid[..., y : y + img_h, x : x + img_w] = images[idx]
-    return grid
-
-def convert_to_pil_image(image, drange=[0,1]):
-    assert image.ndim == 2 or image.ndim == 3
-    if image.ndim == 3:
-        if image.shape[0] == 1:
-            image = image[0] # grayscale CHW => HW
-        else:
-            image = image.transpose(1, 2, 0) # CHW -> HWC
-
-    image = adjust_dynamic_range(image, drange, [0,255])
-    image = np.rint(image).clip(0, 255).astype(np.uint8)
-    fmt = 'RGB' if image.ndim == 3 else 'L'
-    return PIL.Image.fromarray(image, fmt)
-
-def save_image_grid(images, filename, drange=[0,1], grid_size=None):
-    convert_to_pil_image(create_image_grid(images, grid_size), drange).save(filename)
-
-def apply_mirror_augment(minibatch):
-    mask = np.random.rand(minibatch.shape[0]) < 0.5
-    minibatch = np.array(minibatch)
-    minibatch[mask] = minibatch[mask, :, :, ::-1]
-    return minibatch
-
-#----------------------------------------------------------------------------
-# Loading data from previous training runs.
-
-def parse_config_for_previous_run(run_dir):
-    with open(os.path.join(run_dir, 'submit_config.pkl'), 'rb') as f:
-        data = pickle.load(f)
-    data = data.get('run_func_kwargs', {})
-    return dict(train=data, dataset=data.get('dataset_args', {}))
-
-#----------------------------------------------------------------------------
-# Size and contents of the image snapshot grids that are exported
-# periodically during training.
-
-def setup_snapshot_image_grid(training_set,
-    size    = '1080p',      # '1080p' = to be viewed on 1080p display, '4k' = to be viewed on 4k display.
-    layout  = 'random'):    # 'random' = grid contents are selected randomly, 'row_per_class' = each row corresponds to one class label.
-
-    # Select size.
-    gw = 1; gh = 1
-    if size == '1080p':
-        gw = np.clip(1920 // training_set.shape[2], 3, 32)
-        gh = np.clip(1080 // training_set.shape[1], 2, 32)
-    if size == '4k':
-        gw = np.clip(3840 // training_set.shape[2], 7, 32)
-        gh = np.clip(2160 // training_set.shape[1], 4, 32)
-    if size == '8k':
-        gw = np.clip(7680 // training_set.shape[2], 7, 32)
-        gh = np.clip(4320 // training_set.shape[1], 4, 32)
-
-    # Initialize data arrays.
-    reals = np.zeros([gw * gh] + training_set.shape, dtype=training_set.dtype)
-    labels = np.zeros([gw * gh, training_set.label_size], dtype=training_set.label_dtype)
-
-    # Random layout.
-    if layout == 'random':
-        reals[:], labels[:] = training_set.get_minibatch_np(gw * gh)
-
-    # Class-conditional layouts.
-    class_layouts = dict(row_per_class=[gw,1], col_per_class=[1,gh], class4x4=[4,4])
-    if layout in class_layouts:
-        bw, bh = class_layouts[layout]
-        nw = (gw - 1) // bw + 1
-        nh = (gh - 1) // bh + 1
-        blocks = [[] for _i in range(nw * nh)]
-        for _iter in range(1000000):
-            real, label = training_set.get_minibatch_np(1)
-            idx = np.argmax(label[0])
-            while idx < len(blocks) and len(blocks[idx]) >= bw * bh:
-                idx += training_set.label_size
-            if idx < len(blocks):
-                blocks[idx].append((real, label))
-                if all(len(block) >= bw * bh for block in blocks):
-                    break
-        for i, block in enumerate(blocks):
-            for j, (real, label) in enumerate(block):
-                x = (i %  nw) * bw + j %  bw
-                y = (i // nw) * bh + j // bw
-                if x < gw and y < gh:
-                    reals[x + y * gw] = real[0]
-                    labels[x + y * gw] = label[0]
-
-    return (gw, gh), reals, labels
-
-#----------------------------------------------------------------------------

training/networks_stylegan.py

@@ -1,660 +0,0 @@
-# Copyright (c) 2019, NVIDIA Corporation. All rights reserved.
-#
-# This work is made available under the Nvidia Source Code License-NC.
-# To view a copy of this license, visit
-# https://nvlabs.github.io/stylegan2/license.html
-
-"""Network architectures used in the StyleGAN paper."""
-
-import numpy as np
-import tensorflow as tf
-import dnnlib
-import dnnlib.tflib as tflib
-
-# NOTE: Do not import any application-specific modules here!
-# Specify all network parameters as kwargs.
-
-#----------------------------------------------------------------------------
-# Primitive ops for manipulating 4D activation tensors.
-# The gradients of these are not necessary efficient or even meaningful.
-
-def _blur2d(x, f=[1,2,1], normalize=True, flip=False, stride=1):
-    assert x.shape.ndims == 4 and all(dim.value is not None for dim in x.shape[1:])
-    assert isinstance(stride, int) and stride >= 1
-
-    # Finalize filter kernel.
-    f = np.array(f, dtype=np.float32)
-    if f.ndim == 1:
-        f = f[:, np.newaxis] * f[np.newaxis, :]
-    assert f.ndim == 2
-    if normalize:
-        f /= np.sum(f)
-    if flip:
-        f = f[::-1, ::-1]
-    f = f[:, :, np.newaxis, np.newaxis]
-    f = np.tile(f, [1, 1, int(x.shape[1]), 1])
-
-    # No-op => early exit.
-    if f.shape == (1, 1) and f[0,0] == 1:
-        return x
-
-    # Convolve using depthwise_conv2d.
-    orig_dtype = x.dtype
-    x = tf.cast(x, tf.float32)  # tf.nn.depthwise_conv2d() doesn't support fp16
-    f = tf.constant(f, dtype=x.dtype, name='filter')
-    strides = [1, 1, stride, stride]
-    x = tf.nn.depthwise_conv2d(x, f, strides=strides, padding='SAME', data_format='NCHW')
-    x = tf.cast(x, orig_dtype)
-    return x
-
-def _upscale2d(x, factor=2, gain=1):
-    assert x.shape.ndims == 4 and all(dim.value is not None for dim in x.shape[1:])
-    assert isinstance(factor, int) and factor >= 1
-
-    # Apply gain.
-    if gain != 1:
-        x *= gain
-
-    # No-op => early exit.
-    if factor == 1:
-        return x
-
-    # Upscale using tf.tile().
-    s = x.shape
-    x = tf.reshape(x, [-1, s[1], s[2], 1, s[3], 1])
-    x = tf.tile(x, [1, 1, 1, factor, 1, factor])
-    x = tf.reshape(x, [-1, s[1], s[2] * factor, s[3] * factor])
-    return x
-
-def _downscale2d(x, factor=2, gain=1):
-    assert x.shape.ndims == 4 and all(dim.value is not None for dim in x.shape[1:])
-    assert isinstance(factor, int) and factor >= 1
-
-    # 2x2, float32 => downscale using _blur2d().
-    if factor == 2 and x.dtype == tf.float32:
-        f = [np.sqrt(gain) / factor] * factor
-        return _blur2d(x, f=f, normalize=False, stride=factor)
-
-    # Apply gain.
-    if gain != 1:
-        x *= gain
-
-    # No-op => early exit.
-    if factor == 1:
-        return x
-
-    # Large factor => downscale using tf.nn.avg_pool().
-    # NOTE: Requires tf_config['graph_options.place_pruned_graph']=True to work.
-    ksize = [1, 1, factor, factor]
-    return tf.nn.avg_pool(x, ksize=ksize, strides=ksize, padding='VALID', data_format='NCHW')
-
-#----------------------------------------------------------------------------
-# High-level ops for manipulating 4D activation tensors.
-# The gradients of these are meant to be as efficient as possible.
-
-def blur2d(x, f=[1,2,1], normalize=True):
-    with tf.variable_scope('Blur2D'):
-        @tf.custom_gradient
-        def func(x):
-            y = _blur2d(x, f, normalize)
-            @tf.custom_gradient
-            def grad(dy):
-                dx = _blur2d(dy, f, normalize, flip=True)
-                return dx, lambda ddx: _blur2d(ddx, f, normalize)
-            return y, grad
-        return func(x)
-
-def upscale2d(x, factor=2):
-    with tf.variable_scope('Upscale2D'):
-        @tf.custom_gradient
-        def func(x):
-            y = _upscale2d(x, factor)
-            @tf.custom_gradient
-            def grad(dy):
-                dx = _downscale2d(dy, factor, gain=factor**2)
-                return dx, lambda ddx: _upscale2d(ddx, factor)
-            return y, grad
-        return func(x)
-
-def downscale2d(x, factor=2):
-    with tf.variable_scope('Downscale2D'):
-        @tf.custom_gradient
-        def func(x):
-            y = _downscale2d(x, factor)
-            @tf.custom_gradient
-            def grad(dy):
-                dx = _upscale2d(dy, factor, gain=1/factor**2)
-                return dx, lambda ddx: _downscale2d(ddx, factor)
-            return y, grad
-        return func(x)
-
-#----------------------------------------------------------------------------
-# Get/create weight tensor for a convolutional or fully-connected layer.
-
-def get_weight(shape, gain=np.sqrt(2), use_wscale=False, lrmul=1):
-    fan_in = np.prod(shape[:-1]) # [kernel, kernel, fmaps_in, fmaps_out] or [in, out]
-    he_std = gain / np.sqrt(fan_in) # He init
-
-    # Equalized learning rate and custom learning rate multiplier.
-    if use_wscale:
-        init_std = 1.0 / lrmul
-        runtime_coef = he_std * lrmul
-    else:
-        init_std = he_std / lrmul
-        runtime_coef = lrmul
-
-    # Create variable.
-    init = tf.initializers.random_normal(0, init_std)
-    return tf.get_variable('weight', shape=shape, initializer=init) * runtime_coef
-
-#----------------------------------------------------------------------------
-# Fully-connected layer.
-
-def dense(x, fmaps, **kwargs):
-    if len(x.shape) > 2:
-        x = tf.reshape(x, [-1, np.prod([d.value for d in x.shape[1:]])])
-    w = get_weight([x.shape[1].value, fmaps], **kwargs)
-    w = tf.cast(w, x.dtype)
-    return tf.matmul(x, w)
-
-#----------------------------------------------------------------------------
-# Convolutional layer.
-
-def conv2d(x, fmaps, kernel, **kwargs):
-    assert kernel >= 1 and kernel % 2 == 1
-    w = get_weight([kernel, kernel, x.shape[1].value, fmaps], **kwargs)
-    w = tf.cast(w, x.dtype)
-    return tf.nn.conv2d(x, w, strides=[1,1,1,1], padding='SAME', data_format='NCHW')
-
-#----------------------------------------------------------------------------
-# Fused convolution + scaling.
-# Faster and uses less memory than performing the operations separately.
-
-def upscale2d_conv2d(x, fmaps, kernel, fused_scale='auto', **kwargs):
-    assert kernel >= 1 and kernel % 2 == 1
-    assert fused_scale in [True, False, 'auto']
-    if fused_scale == 'auto':
-        fused_scale = min(x.shape[2:]) * 2 >= 128
-
-    # Not fused => call the individual ops directly.
-    if not fused_scale:
-        return conv2d(upscale2d(x), fmaps, kernel, **kwargs)
-
-    # Fused => perform both ops simultaneously using tf.nn.conv2d_transpose().
-    w = get_weight([kernel, kernel, x.shape[1].value, fmaps], **kwargs)
-    w = tf.transpose(w, [0, 1, 3, 2]) # [kernel, kernel, fmaps_out, fmaps_in]
-    w = tf.pad(w, [[1,1], [1,1], [0,0], [0,0]], mode='CONSTANT')
-    w = tf.add_n([w[1:, 1:], w[:-1, 1:], w[1:, :-1], w[:-1, :-1]])
-    w = tf.cast(w, x.dtype)
-    os = [tf.shape(x)[0], fmaps, x.shape[2] * 2, x.shape[3] * 2]
-    return tf.nn.conv2d_transpose(x, w, os, strides=[1,1,2,2], padding='SAME', data_format='NCHW')
-
-def conv2d_downscale2d(x, fmaps, kernel, fused_scale='auto', **kwargs):
-    assert kernel >= 1 and kernel % 2 == 1
-    assert fused_scale in [True, False, 'auto']
-    if fused_scale == 'auto':
-        fused_scale = min(x.shape[2:]) >= 128
-
-    # Not fused => call the individual ops directly.
-    if not fused_scale:
-        return downscale2d(conv2d(x, fmaps, kernel, **kwargs))
-
-    # Fused => perform both ops simultaneously using tf.nn.conv2d().
-    w = get_weight([kernel, kernel, x.shape[1].value, fmaps], **kwargs)
-    w = tf.pad(w, [[1,1], [1,1], [0,0], [0,0]], mode='CONSTANT')
-    w = tf.add_n([w[1:, 1:], w[:-1, 1:], w[1:, :-1], w[:-1, :-1]]) * 0.25
-    w = tf.cast(w, x.dtype)
-    return tf.nn.conv2d(x, w, strides=[1,1,2,2], padding='SAME', data_format='NCHW')
-
-#----------------------------------------------------------------------------
-# Apply bias to the given activation tensor.
-
-def apply_bias(x, lrmul=1):
-    b = tf.get_variable('bias', shape=[x.shape[1]], initializer=tf.initializers.zeros()) * lrmul
-    b = tf.cast(b, x.dtype)
-    if len(x.shape) == 2:
-        return x + b
-    return x + tf.reshape(b, [1, -1, 1, 1])
-
-#----------------------------------------------------------------------------
-# Leaky ReLU activation. More efficient than tf.nn.leaky_relu() and supports FP16.
-
-def leaky_relu(x, alpha=0.2):
-    with tf.variable_scope('LeakyReLU'):
-        alpha = tf.constant(alpha, dtype=x.dtype, name='alpha')
-        @tf.custom_gradient
-        def func(x):
-            y = tf.maximum(x, x * alpha)
-            @tf.custom_gradient
-            def grad(dy):
-                dx = tf.where(y >= 0, dy, dy * alpha)
-                return dx, lambda ddx: tf.where(y >= 0, ddx, ddx * alpha)
-            return y, grad
-        return func(x)
-
-#----------------------------------------------------------------------------
-# Pixelwise feature vector normalization.
-
-def pixel_norm(x, epsilon=1e-8):
-    with tf.variable_scope('PixelNorm'):
-        epsilon = tf.constant(epsilon, dtype=x.dtype, name='epsilon')
-        return x * tf.rsqrt(tf.reduce_mean(tf.square(x), axis=1, keepdims=True) + epsilon)
-
-#----------------------------------------------------------------------------
-# Instance normalization.
-
-def instance_norm(x, epsilon=1e-8):
-    assert len(x.shape) == 4 # NCHW
-    with tf.variable_scope('InstanceNorm'):
-        orig_dtype = x.dtype
-        x = tf.cast(x, tf.float32)
-        x -= tf.reduce_mean(x, axis=[2,3], keepdims=True)
-        epsilon = tf.constant(epsilon, dtype=x.dtype, name='epsilon')
-        x *= tf.rsqrt(tf.reduce_mean(tf.square(x), axis=[2,3], keepdims=True) + epsilon)
-        x = tf.cast(x, orig_dtype)
-        return x
-
-#----------------------------------------------------------------------------
-# Style modulation.
-
-def style_mod(x, dlatent, **kwargs):
-    with tf.variable_scope('StyleMod'):
-        style = apply_bias(dense(dlatent, fmaps=x.shape[1]*2, gain=1, **kwargs))
-        style = tf.reshape(style, [-1, 2, x.shape[1]] + [1] * (len(x.shape) - 2))
-        return x * (style[:,0] + 1) + style[:,1]
-
-#----------------------------------------------------------------------------
-# Noise input.
-
-def apply_noise(x, noise_var=None, randomize_noise=True):
-    assert len(x.shape) == 4 # NCHW
-    with tf.variable_scope('Noise'):
-        if noise_var is None or randomize_noise:
-            noise = tf.random_normal([tf.shape(x)[0], 1, x.shape[2], x.shape[3]], dtype=x.dtype)
-        else:
-            noise = tf.cast(noise_var, x.dtype)
-        weight = tf.get_variable('weight', shape=[x.shape[1].value], initializer=tf.initializers.zeros())
-        return x + noise * tf.reshape(tf.cast(weight, x.dtype), [1, -1, 1, 1])
-
-#----------------------------------------------------------------------------
-# Minibatch standard deviation.
-
-def minibatch_stddev_layer(x, group_size=4, num_new_features=1):
-    with tf.variable_scope('MinibatchStddev'):
-        group_size = tf.minimum(group_size, tf.shape(x)[0])     # Minibatch must be divisible by (or smaller than) group_size.
-        s = x.shape                                             # [NCHW]  Input shape.
-        y = tf.reshape(x, [group_size, -1, num_new_features, s[1]//num_new_features, s[2], s[3]])   # [GMncHW] Split minibatch into M groups of size G. Split channels into n channel groups c.
-        y = tf.cast(y, tf.float32)                              # [GMncHW] Cast to FP32.
-        y -= tf.reduce_mean(y, axis=0, keepdims=True)           # [GMncHW] Subtract mean over group.
-        y = tf.reduce_mean(tf.square(y), axis=0)                # [MncHW]  Calc variance over group.
-        y = tf.sqrt(y + 1e-8)                                   # [MncHW]  Calc stddev over group.
-        y = tf.reduce_mean(y, axis=[2,3,4], keepdims=True)      # [Mn111]  Take average over fmaps and pixels.
-        y = tf.reduce_mean(y, axis=[2])                         # [Mn11] Split channels into c channel groups
-        y = tf.cast(y, x.dtype)                                 # [Mn11]  Cast back to original data type.
-        y = tf.tile(y, [group_size, 1, s[2], s[3]])             # [NnHW]  Replicate over group and pixels.
-        return tf.concat([x, y], axis=1)                        # [NCHW]  Append as new fmap.
-
-#----------------------------------------------------------------------------
-# Style-based generator used in the StyleGAN paper.
-# Composed of two sub-networks (G_mapping and G_synthesis) that are defined below.
-
-def G_style(
-    latents_in,                                     # First input: Latent vectors (Z) [minibatch, latent_size].
-    labels_in,                                      # Second input: Conditioning labels [minibatch, label_size].
-    truncation_psi          = 0.7,                  # Style strength multiplier for the truncation trick. None = disable.
-    truncation_cutoff       = 8,                    # Number of layers for which to apply the truncation trick. None = disable.
-    truncation_psi_val      = None,                 # Value for truncation_psi to use during validation.
-    truncation_cutoff_val   = None,                 # Value for truncation_cutoff to use during validation.
-    dlatent_avg_beta        = 0.995,                # Decay for tracking the moving average of W during training. None = disable.
-    style_mixing_prob       = 0.9,                  # Probability of mixing styles during training. None = disable.
-    is_training             = False,                # Network is under training? Enables and disables specific features.
-    is_validation           = False,                # Network is under validation? Chooses which value to use for truncation_psi.
-    is_template_graph       = False,                # True = template graph constructed by the Network class, False = actual evaluation.
-    components              = dnnlib.EasyDict(),    # Container for sub-networks. Retained between calls.
-    **kwargs):                                      # Arguments for sub-networks (G_mapping and G_synthesis).
-
-    # Validate arguments.
-    assert not is_training or not is_validation
-    assert isinstance(components, dnnlib.EasyDict)
-    if is_validation:
-        truncation_psi = truncation_psi_val
-        truncation_cutoff = truncation_cutoff_val
-    if is_training or (truncation_psi is not None and not tflib.is_tf_expression(truncation_psi) and truncation_psi == 1):
-        truncation_psi = None
-    if is_training or (truncation_cutoff is not None and not tflib.is_tf_expression(truncation_cutoff) and truncation_cutoff <= 0):
-        truncation_cutoff = None
-    if not is_training or (dlatent_avg_beta is not None and not tflib.is_tf_expression(dlatent_avg_beta) and dlatent_avg_beta == 1):
-        dlatent_avg_beta = None
-    if not is_training or (style_mixing_prob is not None and not tflib.is_tf_expression(style_mixing_prob) and style_mixing_prob <= 0):
-        style_mixing_prob = None
-
-    # Setup components.
-    if 'synthesis' not in components:
-        components.synthesis = tflib.Network('G_synthesis', func_name=G_synthesis, **kwargs)
-    num_layers = components.synthesis.input_shape[1]
-    dlatent_size = components.synthesis.input_shape[2]
-    if 'mapping' not in components:
-        components.mapping = tflib.Network('G_mapping', func_name=G_mapping, dlatent_broadcast=num_layers, **kwargs)
-
-    # Setup variables.
-    lod_in = tf.get_variable('lod', initializer=np.float32(0), trainable=False)
-    dlatent_avg = tf.get_variable('dlatent_avg', shape=[dlatent_size], initializer=tf.initializers.zeros(), trainable=False)
-
-    # Evaluate mapping network.
-    dlatents = components.mapping.get_output_for(latents_in, labels_in, **kwargs)
-
-    # Update moving average of W.
-    if dlatent_avg_beta is not None:
-        with tf.variable_scope('DlatentAvg'):
-            batch_avg = tf.reduce_mean(dlatents[:, 0], axis=0)
-            update_op = tf.assign(dlatent_avg, tflib.lerp(batch_avg, dlatent_avg, dlatent_avg_beta))
-            with tf.control_dependencies([update_op]):
-                dlatents = tf.identity(dlatents)
-
-    # Perform style mixing regularization.
-    if style_mixing_prob is not None:
-        with tf.name_scope('StyleMix'):
-            latents2 = tf.random_normal(tf.shape(latents_in))
-            dlatents2 = components.mapping.get_output_for(latents2, labels_in, **kwargs)
-            layer_idx = np.arange(num_layers)[np.newaxis, :, np.newaxis]
-            cur_layers = num_layers - tf.cast(lod_in, tf.int32) * 2
-            mixing_cutoff = tf.cond(
-                tf.random_uniform([], 0.0, 1.0) < style_mixing_prob,
-                lambda: tf.random_uniform([], 1, cur_layers, dtype=tf.int32),
-                lambda: cur_layers)
-            dlatents = tf.where(tf.broadcast_to(layer_idx < mixing_cutoff, tf.shape(dlatents)), dlatents, dlatents2)
-
-    # Apply truncation trick.
-    if truncation_psi is not None and truncation_cutoff is not None:
-        with tf.variable_scope('Truncation'):
-            layer_idx = np.arange(num_layers)[np.newaxis, :, np.newaxis]
-            ones = np.ones(layer_idx.shape, dtype=np.float32)
-            coefs = tf.where(layer_idx < truncation_cutoff, truncation_psi * ones, ones)
-            dlatents = tflib.lerp(dlatent_avg, dlatents, coefs)
-
-    # Evaluate synthesis network.
-    with tf.control_dependencies([tf.assign(components.synthesis.find_var('lod'), lod_in)]):
-        images_out = components.synthesis.get_output_for(dlatents, force_clean_graph=is_template_graph, **kwargs)
-    return tf.identity(images_out, name='images_out')
-
-#----------------------------------------------------------------------------
-# Mapping network used in the StyleGAN paper.
-
-def G_mapping(
-    latents_in,                             # First input: Latent vectors (Z) [minibatch, latent_size].
-    labels_in,                              # Second input: Conditioning labels [minibatch, label_size].
-    latent_size             = 512,          # Latent vector (Z) dimensionality.
-    label_size              = 0,            # Label dimensionality, 0 if no labels.
-    dlatent_size            = 512,          # Disentangled latent (W) dimensionality.
-    dlatent_broadcast       = None,         # Output disentangled latent (W) as [minibatch, dlatent_size] or [minibatch, dlatent_broadcast, dlatent_size].
-    mapping_layers          = 8,            # Number of mapping layers.
-    mapping_fmaps           = 512,          # Number of activations in the mapping layers.
-    mapping_lrmul           = 0.01,         # Learning rate multiplier for the mapping layers.
-    mapping_nonlinearity    = 'lrelu',      # Activation function: 'relu', 'lrelu'.
-    use_wscale              = True,         # Enable equalized learning rate?
-    normalize_latents       = True,         # Normalize latent vectors (Z) before feeding them to the mapping layers?
-    dtype                   = 'float32',    # Data type to use for activations and outputs.
-    **_kwargs):                             # Ignore unrecognized keyword args.
-
-    act, gain = {'relu': (tf.nn.relu, np.sqrt(2)), 'lrelu': (leaky_relu, np.sqrt(2))}[mapping_nonlinearity]
-
-    # Inputs.
-    latents_in.set_shape([None, latent_size])
-    labels_in.set_shape([None, label_size])
-    latents_in = tf.cast(latents_in, dtype)
-    labels_in = tf.cast(labels_in, dtype)
-    x = latents_in
-
-    # Embed labels and concatenate them with latents.
-    if label_size:
-        with tf.variable_scope('LabelConcat'):
-            w = tf.get_variable('weight', shape=[label_size, latent_size], initializer=tf.initializers.random_normal())
-            y = tf.matmul(labels_in, tf.cast(w, dtype))
-            x = tf.concat([x, y], axis=1)
-
-    # Normalize latents.
-    if normalize_latents:
-        x = pixel_norm(x)
-
-    # Mapping layers.
-    for layer_idx in range(mapping_layers):
-        with tf.variable_scope('Dense%d' % layer_idx):
-            fmaps = dlatent_size if layer_idx == mapping_layers - 1 else mapping_fmaps
-            x = dense(x, fmaps=fmaps, gain=gain, use_wscale=use_wscale, lrmul=mapping_lrmul)
-            x = apply_bias(x, lrmul=mapping_lrmul)
-            x = act(x)
-
-    # Broadcast.
-    if dlatent_broadcast is not None:
-        with tf.variable_scope('Broadcast'):
-            x = tf.tile(x[:, np.newaxis], [1, dlatent_broadcast, 1])
-
-    # Output.
-    assert x.dtype == tf.as_dtype(dtype)
-    return tf.identity(x, name='dlatents_out')
-
-#----------------------------------------------------------------------------
-# Synthesis network used in the StyleGAN paper.
-
-def G_synthesis(
-    dlatents_in,                        # Input: Disentangled latents (W) [minibatch, num_layers, dlatent_size].
-    dlatent_size        = 512,          # Disentangled latent (W) dimensionality.
-    num_channels        = 3,            # Number of output color channels.
-    resolution          = 1024,         # Output resolution.
-    fmap_base           = 8192,         # Overall multiplier for the number of feature maps.
-    fmap_decay          = 1.0,          # log2 feature map reduction when doubling the resolution.
-    fmap_max            = 512,          # Maximum number of feature maps in any layer.
-    use_styles          = True,         # Enable style inputs?
-    const_input_layer   = True,         # First layer is a learned constant?
-    use_noise           = True,         # Enable noise inputs?
-    randomize_noise     = True,         # True = randomize noise inputs every time (non-deterministic), False = read noise inputs from variables.
-    nonlinearity        = 'lrelu',      # Activation function: 'relu', 'lrelu'
-    use_wscale          = True,         # Enable equalized learning rate?
-    use_pixel_norm      = False,        # Enable pixelwise feature vector normalization?
-    use_instance_norm   = True,         # Enable instance normalization?
-    dtype               = 'float32',    # Data type to use for activations and outputs.
-    fused_scale         = 'auto',       # True = fused convolution + scaling, False = separate ops, 'auto' = decide automatically.
-    blur_filter         = [1,2,1],      # Low-pass filter to apply when resampling activations. None = no filtering.
-    structure           = 'auto',       # 'fixed' = no progressive growing, 'linear' = human-readable, 'recursive' = efficient, 'auto' = select automatically.
-    is_template_graph   = False,        # True = template graph constructed by the Network class, False = actual evaluation.
-    force_clean_graph   = False,        # True = construct a clean graph that looks nice in TensorBoard, False = default behavior.
-    **_kwargs):                         # Ignore unrecognized keyword args.
-
-    resolution_log2 = int(np.log2(resolution))
-    assert resolution == 2**resolution_log2 and resolution >= 4
-    def nf(stage): return min(int(fmap_base / (2.0 ** (stage * fmap_decay))), fmap_max)
-    def blur(x): return blur2d(x, blur_filter) if blur_filter else x
-    if is_template_graph: force_clean_graph = True
-    if force_clean_graph: randomize_noise = False
-    if structure == 'auto': structure = 'linear' if force_clean_graph else 'recursive'
-    act, gain = {'relu': (tf.nn.relu, np.sqrt(2)), 'lrelu': (leaky_relu, np.sqrt(2))}[nonlinearity]
-    num_layers = resolution_log2 * 2 - 2
-    num_styles = num_layers if use_styles else 1
-    images_out = None
-
-    # Primary inputs.
-    dlatents_in.set_shape([None, num_styles, dlatent_size])
-    dlatents_in = tf.cast(dlatents_in, dtype)
-    lod_in = tf.cast(tf.get_variable('lod', initializer=np.float32(0), trainable=False), dtype)
-
-    # Noise inputs.
-    noise_inputs = []
-    if use_noise:
-        for layer_idx in range(num_layers):
-            res = layer_idx // 2 + 2
-            shape = [1, use_noise, 2**res, 2**res]
-            noise_inputs.append(tf.get_variable('noise%d' % layer_idx, shape=shape, initializer=tf.initializers.random_normal(), trainable=False))
-
-    # Things to do at the end of each layer.
-    def layer_epilogue(x, layer_idx):
-        if use_noise:
-            x = apply_noise(x, noise_inputs[layer_idx], randomize_noise=randomize_noise)
-        x = apply_bias(x)
-        x = act(x)
-        if use_pixel_norm:
-            x = pixel_norm(x)
-        if use_instance_norm:
-            x = instance_norm(x)
-        if use_styles:
-            x = style_mod(x, dlatents_in[:, layer_idx], use_wscale=use_wscale)
-        return x
-
-    # Early layers.
-    with tf.variable_scope('4x4'):
-        if const_input_layer:
-            with tf.variable_scope('Const'):
-                x = tf.get_variable('const', shape=[1, nf(1), 4, 4], initializer=tf.initializers.ones())
-                x = layer_epilogue(tf.tile(tf.cast(x, dtype), [tf.shape(dlatents_in)[0], 1, 1, 1]), 0)
-        else:
-            with tf.variable_scope('Dense'):
-                x = dense(dlatents_in[:, 0], fmaps=nf(1)*16, gain=gain/4, use_wscale=use_wscale) # tweak gain to match the official implementation of Progressing GAN
-                x = layer_epilogue(tf.reshape(x, [-1, nf(1), 4, 4]), 0)
-        with tf.variable_scope('Conv'):
-            x = layer_epilogue(conv2d(x, fmaps=nf(1), kernel=3, gain=gain, use_wscale=use_wscale), 1)
-
-    # Building blocks for remaining layers.
-    def block(res, x): # res = 3..resolution_log2
-        with tf.variable_scope('%dx%d' % (2**res, 2**res)):
-            with tf.variable_scope('Conv0_up'):
-                x = layer_epilogue(blur(upscale2d_conv2d(x, fmaps=nf(res-1), kernel=3, gain=gain, use_wscale=use_wscale, fused_scale=fused_scale)), res*2-4)
-            with tf.variable_scope('Conv1'):
-                x = layer_epilogue(conv2d(x, fmaps=nf(res-1), kernel=3, gain=gain, use_wscale=use_wscale), res*2-3)
-            return x
-    def torgb(res, x): # res = 2..resolution_log2
-        lod = resolution_log2 - res
-        with tf.variable_scope('ToRGB_lod%d' % lod):
-            return apply_bias(conv2d(x, fmaps=num_channels, kernel=1, gain=1, use_wscale=use_wscale))
-
-    # Fixed structure: simple and efficient, but does not support progressive growing.
-    if structure == 'fixed':
-        for res in range(3, resolution_log2 + 1):
-            x = block(res, x)
-        images_out = torgb(resolution_log2, x)
-
-    # Linear structure: simple but inefficient.
-    if structure == 'linear':
-        images_out = torgb(2, x)
-        for res in range(3, resolution_log2 + 1):
-            lod = resolution_log2 - res
-            x = block(res, x)
-            img = torgb(res, x)
-            images_out = upscale2d(images_out)
-            with tf.variable_scope('Grow_lod%d' % lod):
-                images_out = tflib.lerp_clip(img, images_out, lod_in - lod)
-
-    # Recursive structure: complex but efficient.
-    if structure == 'recursive':
-        def cset(cur_lambda, new_cond, new_lambda):
-            return lambda: tf.cond(new_cond, new_lambda, cur_lambda)
-        def grow(x, res, lod):
-            y = block(res, x)
-            img = lambda: upscale2d(torgb(res, y), 2**lod)
-            img = cset(img, (lod_in > lod), lambda: upscale2d(tflib.lerp(torgb(res, y), upscale2d(torgb(res - 1, x)), lod_in - lod), 2**lod))
-            if lod > 0: img = cset(img, (lod_in < lod), lambda: grow(y, res + 1, lod - 1))
-            return img()
-        images_out = grow(x, 3, resolution_log2 - 3)
-
-    assert images_out.dtype == tf.as_dtype(dtype)
-    return tf.identity(images_out, name='images_out')
-
-#----------------------------------------------------------------------------
-# Discriminator used in the StyleGAN paper.
-
-def D_basic(
-    images_in,                          # First input: Images [minibatch, channel, height, width].
-    labels_in,                          # Second input: Labels [minibatch, label_size].
-    num_channels        = 1,            # Number of input color channels. Overridden based on dataset.
-    resolution          = 32,           # Input resolution. Overridden based on dataset.
-    label_size          = 0,            # Dimensionality of the labels, 0 if no labels. Overridden based on dataset.
-    fmap_base           = 8192,         # Overall multiplier for the number of feature maps.
-    fmap_decay          = 1.0,          # log2 feature map reduction when doubling the resolution.
-    fmap_max            = 512,          # Maximum number of feature maps in any layer.
-    nonlinearity        = 'lrelu',      # Activation function: 'relu', 'lrelu',
-    use_wscale          = True,         # Enable equalized learning rate?
-    mbstd_group_size    = 4,            # Group size for the minibatch standard deviation layer, 0 = disable.
-    mbstd_num_features  = 1,            # Number of features for the minibatch standard deviation layer.
-    dtype               = 'float32',    # Data type to use for activations and outputs.
-    fused_scale         = 'auto',       # True = fused convolution + scaling, False = separate ops, 'auto' = decide automatically.
-    blur_filter         = [1,2,1],      # Low-pass filter to apply when resampling activations. None = no filtering.
-    structure           = 'auto',       # 'fixed' = no progressive growing, 'linear' = human-readable, 'recursive' = efficient, 'auto' = select automatically.
-    is_template_graph   = False,        # True = template graph constructed by the Network class, False = actual evaluation.
-    **_kwargs):                         # Ignore unrecognized keyword args.
-
-    resolution_log2 = int(np.log2(resolution))
-    assert resolution == 2**resolution_log2 and resolution >= 4
-    def nf(stage): return min(int(fmap_base / (2.0 ** (stage * fmap_decay))), fmap_max)
-    def blur(x): return blur2d(x, blur_filter) if blur_filter else x
-    if structure == 'auto': structure = 'linear' if is_template_graph else 'recursive'
-    act, gain = {'relu': (tf.nn.relu, np.sqrt(2)), 'lrelu': (leaky_relu, np.sqrt(2))}[nonlinearity]
-
-    images_in.set_shape([None, num_channels, resolution, resolution])
-    labels_in.set_shape([None, label_size])
-    images_in = tf.cast(images_in, dtype)
-    labels_in = tf.cast(labels_in, dtype)
-    lod_in = tf.cast(tf.get_variable('lod', initializer=np.float32(0.0), trainable=False), dtype)
-    scores_out = None
-
-    # Building blocks.
-    def fromrgb(x, res): # res = 2..resolution_log2
-        with tf.variable_scope('FromRGB_lod%d' % (resolution_log2 - res)):
-            return act(apply_bias(conv2d(x, fmaps=nf(res-1), kernel=1, gain=gain, use_wscale=use_wscale)))
-    def block(x, res): # res = 2..resolution_log2
-        with tf.variable_scope('%dx%d' % (2**res, 2**res)):
-            if res >= 3: # 8x8 and up
-                with tf.variable_scope('Conv0'):
-                    x = act(apply_bias(conv2d(x, fmaps=nf(res-1), kernel=3, gain=gain, use_wscale=use_wscale)))
-                with tf.variable_scope('Conv1_down'):
-                    x = act(apply_bias(conv2d_downscale2d(blur(x), fmaps=nf(res-2), kernel=3, gain=gain, use_wscale=use_wscale, fused_scale=fused_scale)))
-            else: # 4x4
-                if mbstd_group_size > 1:
-                    x = minibatch_stddev_layer(x, mbstd_group_size, mbstd_num_features)
-                with tf.variable_scope('Conv'):
-                    x = act(apply_bias(conv2d(x, fmaps=nf(res-1), kernel=3, gain=gain, use_wscale=use_wscale)))
-                with tf.variable_scope('Dense0'):
-                    x = act(apply_bias(dense(x, fmaps=nf(res-2), gain=gain, use_wscale=use_wscale)))
-                with tf.variable_scope('Dense1'):
-                    x = apply_bias(dense(x, fmaps=max(label_size, 1), gain=1, use_wscale=use_wscale))
-            return x
-
-    # Fixed structure: simple and efficient, but does not support progressive growing.
-    if structure == 'fixed':
-        x = fromrgb(images_in, resolution_log2)
-        for res in range(resolution_log2, 2, -1):
-            x = block(x, res)
-        scores_out = block(x, 2)
-
-    # Linear structure: simple but inefficient.
-    if structure == 'linear':
-        img = images_in
-        x = fromrgb(img, resolution_log2)
-        for res in range(resolution_log2, 2, -1):
-            lod = resolution_log2 - res
-            x = block(x, res)
-            img = downscale2d(img)
-            y = fromrgb(img, res - 1)
-            with tf.variable_scope('Grow_lod%d' % lod):
-                x = tflib.lerp_clip(x, y, lod_in - lod)
-        scores_out = block(x, 2)
-
-    # Recursive structure: complex but efficient.
-    if structure == 'recursive':
-        def cset(cur_lambda, new_cond, new_lambda):
-            return lambda: tf.cond(new_cond, new_lambda, cur_lambda)
-        def grow(res, lod):
-            x = lambda: fromrgb(downscale2d(images_in, 2**lod), res)
-            if lod > 0: x = cset(x, (lod_in < lod), lambda: grow(res + 1, lod - 1))
-            x = block(x(), res); y = lambda: x
-            if res > 2: y = cset(y, (lod_in > lod), lambda: tflib.lerp(x, fromrgb(downscale2d(images_in, 2**(lod+1)), res - 1), lod_in - lod))
-            return y()
-        scores_out = grow(2, resolution_log2 - 2)
-
-    # Label conditioning from "Which Training Methods for GANs do actually Converge?"
-    if label_size:
-        with tf.variable_scope('LabelSwitch'):
-            scores_out = tf.reduce_sum(scores_out * labels_in, axis=1, keepdims=True)
-
-    assert scores_out.dtype == tf.as_dtype(dtype)
-    scores_out = tf.identity(scores_out, name='scores_out')
-    return scores_out
-
-#----------------------------------------------------------------------------

training/networks_stylegan2.py

@@ -1,697 +0,0 @@
-# Copyright (c) 2019, NVIDIA Corporation. All rights reserved.
-#
-# This work is made available under the Nvidia Source Code License-NC.
-# To view a copy of this license, visit
-# https://nvlabs.github.io/stylegan2/license.html
-
-"""Network architectures used in the StyleGAN2 paper."""
-
-import numpy as np
-import tensorflow as tf
-import dnnlib
-import dnnlib.tflib as tflib
-from dnnlib.tflib.ops.upfirdn_2d import upsample_2d, downsample_2d, upsample_conv_2d, conv_downsample_2d
-from dnnlib.tflib.ops.fused_bias_act import fused_bias_act
-
-# NOTE: Do not import any application-specific modules here!
-# Specify all network parameters as kwargs.
-
-#----------------------------------------------------------------------------
-# Get/create weight tensor for a convolution or fully-connected layer.
-
-def get_weight(shape, gain=1, use_wscale=True, lrmul=1, weight_var='weight'):
-    fan_in = np.prod(shape[:-1]) # [kernel, kernel, fmaps_in, fmaps_out] or [in, out]
-    he_std = gain / np.sqrt(fan_in) # He init
-
-    # Equalized learning rate and custom learning rate multiplier.
-    if use_wscale:
-        init_std = 1.0 / lrmul
-        runtime_coef = he_std * lrmul
-    else:
-        init_std = he_std / lrmul
-        runtime_coef = lrmul
-
-    # Create variable.
-    init = tf.initializers.random_normal(0, init_std)
-    return tf.get_variable(weight_var, shape=shape, initializer=init) * runtime_coef
-
-#----------------------------------------------------------------------------
-# Fully-connected layer.
-
-def dense_layer(x, fmaps, gain=1, use_wscale=True, lrmul=1, weight_var='weight'):
-    if len(x.shape) > 2:
-        x = tf.reshape(x, [-1, np.prod([d.value for d in x.shape[1:]])])
-    w = get_weight([x.shape[1].value, fmaps], gain=gain, use_wscale=use_wscale, lrmul=lrmul, weight_var=weight_var)
-    w = tf.cast(w, x.dtype)
-    return tf.matmul(x, w)
-
-#----------------------------------------------------------------------------
-# Convolution layer with optional upsampling or downsampling.
-
-def conv2d_layer(x, fmaps, kernel, up=False, down=False, resample_kernel=None, gain=1, use_wscale=True, lrmul=1, weight_var='weight'):
-    assert not (up and down)
-    assert kernel >= 1 and kernel % 2 == 1
-    w = get_weight([kernel, kernel, x.shape[1].value, fmaps], gain=gain, use_wscale=use_wscale, lrmul=lrmul, weight_var=weight_var)
-    if up:
-        x = upsample_conv_2d(x, tf.cast(w, x.dtype), data_format='NCHW', k=resample_kernel)
-    elif down:
-        x = conv_downsample_2d(x, tf.cast(w, x.dtype), data_format='NCHW', k=resample_kernel)
-    else:
-        x = tf.nn.conv2d(x, tf.cast(w, x.dtype), data_format='NCHW', strides=[1,1,1,1], padding='SAME')
-    return x
-
-#----------------------------------------------------------------------------
-# Apply bias and activation func.
-
-def apply_bias_act(x, act='linear', alpha=None, gain=None, lrmul=1, bias_var='bias'):
-    b = tf.get_variable(bias_var, shape=[x.shape[1]], initializer=tf.initializers.zeros()) * lrmul
-    return fused_bias_act(x, b=tf.cast(b, x.dtype), act=act, alpha=alpha, gain=gain)
-
-#----------------------------------------------------------------------------
-# Naive upsampling (nearest neighbor) and downsampling (average pooling).
-
-def naive_upsample_2d(x, factor=2):
-    with tf.variable_scope('NaiveUpsample'):
-        _N, C, H, W = x.shape.as_list()
-        x = tf.reshape(x, [-1, C, H, 1, W, 1])
-        x = tf.tile(x, [1, 1, 1, factor, 1, factor])
-        return tf.reshape(x, [-1, C, H * factor, W * factor])
-
-def naive_downsample_2d(x, factor=2):
-    with tf.variable_scope('NaiveDownsample'):
-        _N, C, H, W = x.shape.as_list()
-        x = tf.reshape(x, [-1, C, H // factor, factor, W // factor, factor])
-        return tf.reduce_mean(x, axis=[3,5])
-
-#----------------------------------------------------------------------------
-# Modulated convolution layer.
-
-def modulated_conv2d_layer(x, y, fmaps, kernel, up=False, down=False, demodulate=True, resample_kernel=None, gain=1, use_wscale=True, lrmul=1, fused_modconv=True, weight_var='weight', mod_weight_var='mod_weight', mod_bias_var='mod_bias'):
-    assert not (up and down)
-    assert kernel >= 1 and kernel % 2 == 1
-
-    # Get weight.
-    w = get_weight([kernel, kernel, x.shape[1].value, fmaps], gain=gain, use_wscale=use_wscale, lrmul=lrmul, weight_var=weight_var)
-    ww = w[np.newaxis] # [BkkIO] Introduce minibatch dimension.
-
-    # Modulate.
-    s = dense_layer(y, fmaps=x.shape[1].value, weight_var=mod_weight_var) # [BI] Transform incoming W to style.
-    s = apply_bias_act(s, bias_var=mod_bias_var) + 1 # [BI] Add bias (initially 1).
-    ww *= tf.cast(s[:, np.newaxis, np.newaxis, :, np.newaxis], w.dtype) # [BkkIO] Scale input feature maps.
-
-    # Demodulate.
-    if demodulate:
-        d = tf.rsqrt(tf.reduce_sum(tf.square(ww), axis=[1,2,3]) + 1e-8) # [BO] Scaling factor.
-        ww *= d[:, np.newaxis, np.newaxis, np.newaxis, :] # [BkkIO] Scale output feature maps.
-
-    # Reshape/scale input.
-    if fused_modconv:
-        x = tf.reshape(x, [1, -1, x.shape[2], x.shape[3]]) # Fused => reshape minibatch to convolution groups.
-        w = tf.reshape(tf.transpose(ww, [1, 2, 3, 0, 4]), [ww.shape[1], ww.shape[2], ww.shape[3], -1])
-    else:
-        x *= tf.cast(s[:, :, np.newaxis, np.newaxis], x.dtype) # [BIhw] Not fused => scale input activations.
-
-    # Convolution with optional up/downsampling.
-    if up:
-        x = upsample_conv_2d(x, tf.cast(w, x.dtype), data_format='NCHW', k=resample_kernel)
-    elif down:
-        x = conv_downsample_2d(x, tf.cast(w, x.dtype), data_format='NCHW', k=resample_kernel)
-    else:
-        x = tf.nn.conv2d(x, tf.cast(w, x.dtype), data_format='NCHW', strides=[1,1,1,1], padding='SAME')
-
-    # Reshape/scale output.
-    if fused_modconv:
-        x = tf.reshape(x, [-1, fmaps, x.shape[2], x.shape[3]]) # Fused => reshape convolution groups back to minibatch.
-    elif demodulate:
-        x *= tf.cast(d[:, :, np.newaxis, np.newaxis], x.dtype) # [BOhw] Not fused => scale output activations.
-    return x
-
-#----------------------------------------------------------------------------
-# Minibatch standard deviation layer.
-
-def minibatch_stddev_layer(x, group_size=4, num_new_features=1):
-    group_size = tf.minimum(group_size, tf.shape(x)[0])     # Minibatch must be divisible by (or smaller than) group_size.
-    s = x.shape                                             # [NCHW]  Input shape.
-    y = tf.reshape(x, [group_size, -1, num_new_features, s[1]//num_new_features, s[2], s[3]])   # [GMncHW] Split minibatch into M groups of size G. Split channels into n channel groups c.
-    y = tf.cast(y, tf.float32)                              # [GMncHW] Cast to FP32.
-    y -= tf.reduce_mean(y, axis=0, keepdims=True)           # [GMncHW] Subtract mean over group.
-    y = tf.reduce_mean(tf.square(y), axis=0)                # [MncHW]  Calc variance over group.
-    y = tf.sqrt(y + 1e-8)                                   # [MncHW]  Calc stddev over group.
-    y = tf.reduce_mean(y, axis=[2,3,4], keepdims=True)      # [Mn111]  Take average over fmaps and pixels.
-    y = tf.reduce_mean(y, axis=[2])                         # [Mn11] Split channels into c channel groups
-    y = tf.cast(y, x.dtype)                                 # [Mn11]  Cast back to original data type.
-    y = tf.tile(y, [group_size, 1, s[2], s[3]])             # [NnHW]  Replicate over group and pixels.
-    return tf.concat([x, y], axis=1)                        # [NCHW]  Append as new fmap.
-
-#----------------------------------------------------------------------------
-# Main generator network.
-# Composed of two sub-networks (mapping and synthesis) that are defined below.
-# Used in configs B-F (Table 1).
-
-def G_main(
-    latents_in,                                         # First input: Latent vectors (Z) [minibatch, latent_size].
-    labels_in,                                          # Second input: Conditioning labels [minibatch, label_size].
-    truncation_psi          = 0.5,                      # Style strength multiplier for the truncation trick. None = disable.
-    truncation_cutoff       = None,                     # Number of layers for which to apply the truncation trick. None = disable.
-    truncation_psi_val      = None,                     # Value for truncation_psi to use during validation.
-    truncation_cutoff_val   = None,                     # Value for truncation_cutoff to use during validation.
-    dlatent_avg_beta        = 0.995,                    # Decay for tracking the moving average of W during training. None = disable.
-    style_mixing_prob       = 0.9,                      # Probability of mixing styles during training. None = disable.
-    is_training             = False,                    # Network is under training? Enables and disables specific features.
-    is_validation           = False,                    # Network is under validation? Chooses which value to use for truncation_psi.
-    return_dlatents         = False,                    # Return dlatents in addition to the images?
-    is_template_graph       = False,                    # True = template graph constructed by the Network class, False = actual evaluation.
-    components              = dnnlib.EasyDict(),        # Container for sub-networks. Retained between calls.
-    mapping_func            = 'G_mapping',              # Build func name for the mapping network.
-    synthesis_func          = 'G_synthesis_stylegan2',  # Build func name for the synthesis network.
-    **kwargs):                                          # Arguments for sub-networks (mapping and synthesis).
-
-    # Validate arguments.
-    assert not is_training or not is_validation
-    assert isinstance(components, dnnlib.EasyDict)
-    if is_validation:
-        truncation_psi = truncation_psi_val
-        truncation_cutoff = truncation_cutoff_val
-    if is_training or (truncation_psi is not None and not tflib.is_tf_expression(truncation_psi) and truncation_psi == 1):
-        truncation_psi = None
-    if is_training:
-        truncation_cutoff = None
-    if not is_training or (dlatent_avg_beta is not None and not tflib.is_tf_expression(dlatent_avg_beta) and dlatent_avg_beta == 1):
-        dlatent_avg_beta = None
-    if not is_training or (style_mixing_prob is not None and not tflib.is_tf_expression(style_mixing_prob) and style_mixing_prob <= 0):
-        style_mixing_prob = None
-
-    # Setup components.
-    if 'synthesis' not in components:
-        components.synthesis = tflib.Network('G_synthesis', func_name=globals()[synthesis_func], **kwargs)
-    num_layers = components.synthesis.input_shape[1]
-    dlatent_size = components.synthesis.input_shape[2]
-    if 'mapping' not in components:
-        components.mapping = tflib.Network('G_mapping', func_name=globals()[mapping_func], dlatent_broadcast=num_layers, **kwargs)
-
-    # Setup variables.
-    lod_in = tf.get_variable('lod', initializer=np.float32(0), trainable=False)
-    dlatent_avg = tf.get_variable('dlatent_avg', shape=[dlatent_size], initializer=tf.initializers.zeros(), trainable=False)
-
-    # Evaluate mapping network.
-    dlatents = components.mapping.get_output_for(latents_in, labels_in, is_training=is_training, **kwargs)
-    dlatents = tf.cast(dlatents, tf.float32)
-
-    # Update moving average of W.
-    if dlatent_avg_beta is not None:
-        with tf.variable_scope('DlatentAvg'):
-            batch_avg = tf.reduce_mean(dlatents[:, 0], axis=0)
-            update_op = tf.assign(dlatent_avg, tflib.lerp(batch_avg, dlatent_avg, dlatent_avg_beta))
-            with tf.control_dependencies([update_op]):
-                dlatents = tf.identity(dlatents)
-
-    # Perform style mixing regularization.
-    if style_mixing_prob is not None:
-        with tf.variable_scope('StyleMix'):
-            latents2 = tf.random_normal(tf.shape(latents_in))
-            dlatents2 = components.mapping.get_output_for(latents2, labels_in, is_training=is_training, **kwargs)
-            dlatents2 = tf.cast(dlatents2, tf.float32)
-            layer_idx = np.arange(num_layers)[np.newaxis, :, np.newaxis]
-            cur_layers = num_layers - tf.cast(lod_in, tf.int32) * 2
-            mixing_cutoff = tf.cond(
-                tf.random_uniform([], 0.0, 1.0) < style_mixing_prob,
-                lambda: tf.random_uniform([], 1, cur_layers, dtype=tf.int32),
-                lambda: cur_layers)
-            dlatents = tf.where(tf.broadcast_to(layer_idx < mixing_cutoff, tf.shape(dlatents)), dlatents, dlatents2)
-
-    # Apply truncation trick.
-    if truncation_psi is not None:
-        with tf.variable_scope('Truncation'):
-            layer_idx = np.arange(num_layers)[np.newaxis, :, np.newaxis]
-            layer_psi = np.ones(layer_idx.shape, dtype=np.float32)
-            if truncation_cutoff is None:
-                layer_psi *= truncation_psi
-            else:
-                layer_psi = tf.where(layer_idx < truncation_cutoff, layer_psi * truncation_psi, layer_psi)
-            dlatents = tflib.lerp(dlatent_avg, dlatents, layer_psi)
-
-    # Evaluate synthesis network.
-    deps = []
-    if 'lod' in components.synthesis.vars:
-        deps.append(tf.assign(components.synthesis.vars['lod'], lod_in))
-    with tf.control_dependencies(deps):
-        images_out = components.synthesis.get_output_for(dlatents, is_training=is_training, force_clean_graph=is_template_graph, **kwargs)
-
-    # Return requested outputs.
-    images_out = tf.identity(images_out, name='images_out')
-    if return_dlatents:
-        return images_out, dlatents
-    return images_out
-
-#----------------------------------------------------------------------------
-# Mapping network.
-# Transforms the input latent code (z) to the disentangled latent code (w).
-# Used in configs B-F (Table 1).
-
-def G_mapping(
-    latents_in,                             # First input: Latent vectors (Z) [minibatch, latent_size].
-    labels_in,                              # Second input: Conditioning labels [minibatch, label_size].
-    latent_size             = 512,          # Latent vector (Z) dimensionality.
-    label_size              = 0,            # Label dimensionality, 0 if no labels.
-    dlatent_size            = 512,          # Disentangled latent (W) dimensionality.
-    dlatent_broadcast       = None,         # Output disentangled latent (W) as [minibatch, dlatent_size] or [minibatch, dlatent_broadcast, dlatent_size].
-    mapping_layers          = 8,            # Number of mapping layers.
-    mapping_fmaps           = 512,          # Number of activations in the mapping layers.
-    mapping_lrmul           = 0.01,         # Learning rate multiplier for the mapping layers.
-    mapping_nonlinearity    = 'lrelu',      # Activation function: 'relu', 'lrelu', etc.
-    normalize_latents       = True,         # Normalize latent vectors (Z) before feeding them to the mapping layers?
-    dtype                   = 'float32',    # Data type to use for activations and outputs.
-    **_kwargs):                             # Ignore unrecognized keyword args.
-
-    act = mapping_nonlinearity
-
-    # Inputs.
-    latents_in.set_shape([None, latent_size])
-    labels_in.set_shape([None, label_size])
-    latents_in = tf.cast(latents_in, dtype)
-    labels_in = tf.cast(labels_in, dtype)
-    x = latents_in
-
-    # Embed labels and concatenate them with latents.
-    if label_size:
-        with tf.variable_scope('LabelConcat'):
-            w = tf.get_variable('weight', shape=[label_size, latent_size], initializer=tf.initializers.random_normal())
-            y = tf.matmul(labels_in, tf.cast(w, dtype))
-            x = tf.concat([x, y], axis=1)
-
-    # Normalize latents.
-    if normalize_latents:
-        with tf.variable_scope('Normalize'):
-            x *= tf.rsqrt(tf.reduce_mean(tf.square(x), axis=1, keepdims=True) + 1e-8)
-
-    # Mapping layers.
-    for layer_idx in range(mapping_layers):
-        with tf.variable_scope('Dense%d' % layer_idx):
-            fmaps = dlatent_size if layer_idx == mapping_layers - 1 else mapping_fmaps
-            x = apply_bias_act(dense_layer(x, fmaps=fmaps, lrmul=mapping_lrmul), act=act, lrmul=mapping_lrmul)
-
-    # Broadcast.
-    if dlatent_broadcast is not None:
-        with tf.variable_scope('Broadcast'):
-            x = tf.tile(x[:, np.newaxis], [1, dlatent_broadcast, 1])
-
-    # Output.
-    assert x.dtype == tf.as_dtype(dtype)
-    return tf.identity(x, name='dlatents_out')
-
-#----------------------------------------------------------------------------
-# StyleGAN synthesis network with revised architecture (Figure 2d).
-# Implements progressive growing, but no skip connections or residual nets (Figure 7).
-# Used in configs B-D (Table 1).
-
-def G_synthesis_stylegan_revised(
-    dlatents_in,                        # Input: Disentangled latents (W) [minibatch, num_layers, dlatent_size].
-    dlatent_size        = 512,          # Disentangled latent (W) dimensionality.
-    num_channels        = 3,            # Number of output color channels.
-    resolution          = 1024,         # Output resolution.
-    fmap_base           = 16 << 10,     # Overall multiplier for the number of feature maps.
-    fmap_decay          = 1.0,          # log2 feature map reduction when doubling the resolution.
-    fmap_min            = 1,            # Minimum number of feature maps in any layer.
-    fmap_max            = 512,          # Maximum number of feature maps in any layer.
-    randomize_noise     = True,         # True = randomize noise inputs every time (non-deterministic), False = read noise inputs from variables.
-    nonlinearity        = 'lrelu',      # Activation function: 'relu', 'lrelu', etc.
-    dtype               = 'float32',    # Data type to use for activations and outputs.
-    resample_kernel     = [1,3,3,1],    # Low-pass filter to apply when resampling activations. None = no filtering.
-    fused_modconv       = True,         # Implement modulated_conv2d_layer() as a single fused op?
-    structure           = 'auto',       # 'fixed' = no progressive growing, 'linear' = human-readable, 'recursive' = efficient, 'auto' = select automatically.
-    is_template_graph   = False,        # True = template graph constructed by the Network class, False = actual evaluation.
-    force_clean_graph   = False,        # True = construct a clean graph that looks nice in TensorBoard, False = default behavior.
-    **_kwargs):                         # Ignore unrecognized keyword args.
-
-    resolution_log2 = int(np.log2(resolution))
-    assert resolution == 2**resolution_log2 and resolution >= 4
-    def nf(stage): return np.clip(int(fmap_base / (2.0 ** (stage * fmap_decay))), fmap_min, fmap_max)
-    if is_template_graph: force_clean_graph = True
-    if force_clean_graph: randomize_noise = False
-    if structure == 'auto': structure = 'linear' if force_clean_graph else 'recursive'
-    act = nonlinearity
-    num_layers = resolution_log2 * 2 - 2
-    images_out = None
-
-    # Primary inputs.
-    dlatents_in.set_shape([None, num_layers, dlatent_size])
-    dlatents_in = tf.cast(dlatents_in, dtype)
-    lod_in = tf.cast(tf.get_variable('lod', initializer=np.float32(0), trainable=False), dtype)
-
-    # Noise inputs.
-    noise_inputs = []
-    for layer_idx in range(num_layers - 1):
-        res = (layer_idx + 5) // 2
-        shape = [1, 1, 2**res, 2**res]
-        noise_inputs.append(tf.get_variable('noise%d' % layer_idx, shape=shape, initializer=tf.initializers.random_normal(), trainable=False))
-
-    # Single convolution layer with all the bells and whistles.
-    def layer(x, layer_idx, fmaps, kernel, up=False):
-        x = modulated_conv2d_layer(x, dlatents_in[:, layer_idx], fmaps=fmaps, kernel=kernel, up=up, resample_kernel=resample_kernel, fused_modconv=fused_modconv)
-        if randomize_noise:
-            noise = tf.random_normal([tf.shape(x)[0], 1, x.shape[2], x.shape[3]], dtype=x.dtype)
-        else:
-            noise = tf.cast(noise_inputs[layer_idx], x.dtype)
-        noise_strength = tf.get_variable('noise_strength', shape=[], initializer=tf.initializers.zeros())
-        x += noise * tf.cast(noise_strength, x.dtype)
-        return apply_bias_act(x, act=act)
-
-    # Early layers.
-    with tf.variable_scope('4x4'):
-        with tf.variable_scope('Const'):
-            x = tf.get_variable('const', shape=[1, nf(1), 4, 4], initializer=tf.initializers.random_normal())
-            x = tf.tile(tf.cast(x, dtype), [tf.shape(dlatents_in)[0], 1, 1, 1])
-        with tf.variable_scope('Conv'):
-            x = layer(x, layer_idx=0, fmaps=nf(1), kernel=3)
-
-    # Building blocks for remaining layers.
-    def block(res, x): # res = 3..resolution_log2
-        with tf.variable_scope('%dx%d' % (2**res, 2**res)):
-            with tf.variable_scope('Conv0_up'):
-                x = layer(x, layer_idx=res*2-5, fmaps=nf(res-1), kernel=3, up=True)
-            with tf.variable_scope('Conv1'):
-                x = layer(x, layer_idx=res*2-4, fmaps=nf(res-1), kernel=3)
-            return x
-    def torgb(res, x): # res = 2..resolution_log2
-        with tf.variable_scope('ToRGB_lod%d' % (resolution_log2 - res)):
-            return apply_bias_act(modulated_conv2d_layer(x, dlatents_in[:, res*2-3], fmaps=num_channels, kernel=1, demodulate=False, fused_modconv=fused_modconv))
-
-    # Fixed structure: simple and efficient, but does not support progressive growing.
-    if structure == 'fixed':
-        for res in range(3, resolution_log2 + 1):
-            x = block(res, x)
-        images_out = torgb(resolution_log2, x)
-
-    # Linear structure: simple but inefficient.
-    if structure == 'linear':
-        images_out = torgb(2, x)
-        for res in range(3, resolution_log2 + 1):
-            lod = resolution_log2 - res
-            x = block(res, x)
-            img = torgb(res, x)
-            with tf.variable_scope('Upsample_lod%d' % lod):
-                images_out = upsample_2d(images_out)
-            with tf.variable_scope('Grow_lod%d' % lod):
-                images_out = tflib.lerp_clip(img, images_out, lod_in - lod)
-
-    # Recursive structure: complex but efficient.
-    if structure == 'recursive':
-        def cset(cur_lambda, new_cond, new_lambda):
-            return lambda: tf.cond(new_cond, new_lambda, cur_lambda)
-        def grow(x, res, lod):
-            y = block(res, x)
-            img = lambda: naive_upsample_2d(torgb(res, y), factor=2**lod)
-            img = cset(img, (lod_in > lod), lambda: naive_upsample_2d(tflib.lerp(torgb(res, y), upsample_2d(torgb(res - 1, x)), lod_in - lod), factor=2**lod))
-            if lod > 0: img = cset(img, (lod_in < lod), lambda: grow(y, res + 1, lod - 1))
-            return img()
-        images_out = grow(x, 3, resolution_log2 - 3)
-
-    assert images_out.dtype == tf.as_dtype(dtype)
-    return tf.identity(images_out, name='images_out')
-
-#----------------------------------------------------------------------------
-# StyleGAN2 synthesis network (Figure 7).
-# Implements skip connections and residual nets (Figure 7), but no progressive growing.
-# Used in configs E-F (Table 1).
-
-def G_synthesis_stylegan2(
-    dlatents_in,                        # Input: Disentangled latents (W) [minibatch, num_layers, dlatent_size].
-    dlatent_size        = 512,          # Disentangled latent (W) dimensionality.
-    num_channels        = 3,            # Number of output color channels.
-    resolution          = 1024,         # Output resolution.
-    fmap_base           = 16 << 10,     # Overall multiplier for the number of feature maps.
-    fmap_decay          = 1.0,          # log2 feature map reduction when doubling the resolution.
-    fmap_min            = 1,            # Minimum number of feature maps in any layer.
-    fmap_max            = 512,          # Maximum number of feature maps in any layer.
-    randomize_noise     = True,         # True = randomize noise inputs every time (non-deterministic), False = read noise inputs from variables.
-    architecture        = 'skip',       # Architecture: 'orig', 'skip', 'resnet'.
-    nonlinearity        = 'lrelu',      # Activation function: 'relu', 'lrelu', etc.
-    dtype               = 'float32',    # Data type to use for activations and outputs.
-    resample_kernel     = [1,3,3,1],    # Low-pass filter to apply when resampling activations. None = no filtering.
-    fused_modconv       = True,         # Implement modulated_conv2d_layer() as a single fused op?
-    **_kwargs):                         # Ignore unrecognized keyword args.
-
-    resolution_log2 = int(np.log2(resolution))
-    assert resolution == 2**resolution_log2 and resolution >= 4
-    def nf(stage): return np.clip(int(fmap_base / (2.0 ** (stage * fmap_decay))), fmap_min, fmap_max)
-    assert architecture in ['orig', 'skip', 'resnet']
-    act = nonlinearity
-    num_layers = resolution_log2 * 2 - 2
-    images_out = None
-
-    # Primary inputs.
-    dlatents_in.set_shape([None, num_layers, dlatent_size])
-    dlatents_in = tf.cast(dlatents_in, dtype)
-
-    # Noise inputs.
-    noise_inputs = []
-    for layer_idx in range(num_layers - 1):
-        res = (layer_idx + 5) // 2
-        shape = [1, 1, 2**res, 2**res]
-        noise_inputs.append(tf.get_variable('noise%d' % layer_idx, shape=shape, initializer=tf.initializers.random_normal(), trainable=False))
-
-    # Single convolution layer with all the bells and whistles.
-    def layer(x, layer_idx, fmaps, kernel, up=False):
-        x = modulated_conv2d_layer(x, dlatents_in[:, layer_idx], fmaps=fmaps, kernel=kernel, up=up, resample_kernel=resample_kernel, fused_modconv=fused_modconv)
-        if randomize_noise:
-            noise = tf.random_normal([tf.shape(x)[0], 1, x.shape[2], x.shape[3]], dtype=x.dtype)
-        else:
-            noise = tf.cast(noise_inputs[layer_idx], x.dtype)
-        noise_strength = tf.get_variable('noise_strength', shape=[], initializer=tf.initializers.zeros())
-        x += noise * tf.cast(noise_strength, x.dtype)
-        return apply_bias_act(x, act=act)
-
-    # Building blocks for main layers.
-    def block(x, res): # res = 3..resolution_log2
-        t = x
-        with tf.variable_scope('Conv0_up'):
-            x = layer(x, layer_idx=res*2-5, fmaps=nf(res-1), kernel=3, up=True)
-        with tf.variable_scope('Conv1'):
-            x = layer(x, layer_idx=res*2-4, fmaps=nf(res-1), kernel=3)
-        if architecture == 'resnet':
-            with tf.variable_scope('Skip'):
-                t = conv2d_layer(t, fmaps=nf(res-1), kernel=1, up=True, resample_kernel=resample_kernel)
-                x = (x + t) * (1 / np.sqrt(2))
-        return x
-    def upsample(y):
-        with tf.variable_scope('Upsample'):
-            return upsample_2d(y, k=resample_kernel)
-    def torgb(x, y, res): # res = 2..resolution_log2
-        with tf.variable_scope('ToRGB'):
-            t = apply_bias_act(modulated_conv2d_layer(x, dlatents_in[:, res*2-3], fmaps=num_channels, kernel=1, demodulate=False, fused_modconv=fused_modconv))
-            return t if y is None else y + t
-
-    # Early layers.
-    y = None
-    with tf.variable_scope('4x4'):
-        with tf.variable_scope('Const'):
-            x = tf.get_variable('const', shape=[1, nf(1), 4, 4], initializer=tf.initializers.random_normal())
-            x = tf.tile(tf.cast(x, dtype), [tf.shape(dlatents_in)[0], 1, 1, 1])
-        with tf.variable_scope('Conv'):
-            x = layer(x, layer_idx=0, fmaps=nf(1), kernel=3)
-        if architecture == 'skip':
-            y = torgb(x, y, 2)
-
-    # Main layers.
-    for res in range(3, resolution_log2 + 1):
-        with tf.variable_scope('%dx%d' % (2**res, 2**res)):
-            x = block(x, res)
-            if architecture == 'skip':
-                y = upsample(y)
-            if architecture == 'skip' or res == resolution_log2:
-                y = torgb(x, y, res)
-    images_out = y
-
-    assert images_out.dtype == tf.as_dtype(dtype)
-    return tf.identity(images_out, name='images_out')
-
-#----------------------------------------------------------------------------
-# Original StyleGAN discriminator.
-# Used in configs B-D (Table 1).
-
-def D_stylegan(
-    images_in,                          # First input: Images [minibatch, channel, height, width].
-    labels_in,                          # Second input: Labels [minibatch, label_size].
-    num_channels        = 3,            # Number of input color channels. Overridden based on dataset.
-    resolution          = 1024,         # Input resolution. Overridden based on dataset.
-    label_size          = 0,            # Dimensionality of the labels, 0 if no labels. Overridden based on dataset.
-    fmap_base           = 16 << 10,     # Overall multiplier for the number of feature maps.
-    fmap_decay          = 1.0,          # log2 feature map reduction when doubling the resolution.
-    fmap_min            = 1,            # Minimum number of feature maps in any layer.
-    fmap_max            = 512,          # Maximum number of feature maps in any layer.
-    nonlinearity        = 'lrelu',      # Activation function: 'relu', 'lrelu', etc.
-    mbstd_group_size    = 4,            # Group size for the minibatch standard deviation layer, 0 = disable.
-    mbstd_num_features  = 1,            # Number of features for the minibatch standard deviation layer.
-    dtype               = 'float32',    # Data type to use for activations and outputs.
-    resample_kernel     = [1,3,3,1],    # Low-pass filter to apply when resampling activations. None = no filtering.
-    structure           = 'auto',       # 'fixed' = no progressive growing, 'linear' = human-readable, 'recursive' = efficient, 'auto' = select automatically.
-    is_template_graph   = False,        # True = template graph constructed by the Network class, False = actual evaluation.
-    **_kwargs):                         # Ignore unrecognized keyword args.
-
-    resolution_log2 = int(np.log2(resolution))
-    assert resolution == 2**resolution_log2 and resolution >= 4
-    def nf(stage): return np.clip(int(fmap_base / (2.0 ** (stage * fmap_decay))), fmap_min, fmap_max)
-    if structure == 'auto': structure = 'linear' if is_template_graph else 'recursive'
-    act = nonlinearity
-
-    images_in.set_shape([None, num_channels, resolution, resolution])
-    labels_in.set_shape([None, label_size])
-    images_in = tf.cast(images_in, dtype)
-    labels_in = tf.cast(labels_in, dtype)
-    lod_in = tf.cast(tf.get_variable('lod', initializer=np.float32(0.0), trainable=False), dtype)
-
-    # Building blocks for spatial layers.
-    def fromrgb(x, res): # res = 2..resolution_log2
-        with tf.variable_scope('FromRGB_lod%d' % (resolution_log2 - res)):
-            return apply_bias_act(conv2d_layer(x, fmaps=nf(res-1), kernel=1), act=act)
-    def block(x, res): # res = 2..resolution_log2
-        with tf.variable_scope('%dx%d' % (2**res, 2**res)):
-            with tf.variable_scope('Conv0'):
-                x = apply_bias_act(conv2d_layer(x, fmaps=nf(res-1), kernel=3), act=act)
-            with tf.variable_scope('Conv1_down'):
-                x = apply_bias_act(conv2d_layer(x, fmaps=nf(res-2), kernel=3, down=True, resample_kernel=resample_kernel), act=act)
-            return x
-
-    # Fixed structure: simple and efficient, but does not support progressive growing.
-    if structure == 'fixed':
-        x = fromrgb(images_in, resolution_log2)
-        for res in range(resolution_log2, 2, -1):
-            x = block(x, res)
-
-    # Linear structure: simple but inefficient.
-    if structure == 'linear':
-        img = images_in
-        x = fromrgb(img, resolution_log2)
-        for res in range(resolution_log2, 2, -1):
-            lod = resolution_log2 - res
-            x = block(x, res)
-            with tf.variable_scope('Downsample_lod%d' % lod):
-                img = downsample_2d(img)
-            y = fromrgb(img, res - 1)
-            with tf.variable_scope('Grow_lod%d' % lod):
-                x = tflib.lerp_clip(x, y, lod_in - lod)
-
-    # Recursive structure: complex but efficient.
-    if structure == 'recursive':
-        def cset(cur_lambda, new_cond, new_lambda):
-            return lambda: tf.cond(new_cond, new_lambda, cur_lambda)
-        def grow(res, lod):
-            x = lambda: fromrgb(naive_downsample_2d(images_in, factor=2**lod), res)
-            if lod > 0: x = cset(x, (lod_in < lod), lambda: grow(res + 1, lod - 1))
-            x = block(x(), res); y = lambda: x
-            y = cset(y, (lod_in > lod), lambda: tflib.lerp(x, fromrgb(naive_downsample_2d(images_in, factor=2**(lod+1)), res - 1), lod_in - lod))
-            return y()
-        x = grow(3, resolution_log2 - 3)
-
-    # Final layers at 4x4 resolution.
-    with tf.variable_scope('4x4'):
-        if mbstd_group_size > 1:
-            with tf.variable_scope('MinibatchStddev'):
-                x = minibatch_stddev_layer(x, mbstd_group_size, mbstd_num_features)
-        with tf.variable_scope('Conv'):
-            x = apply_bias_act(conv2d_layer(x, fmaps=nf(1), kernel=3), act=act)
-        with tf.variable_scope('Dense0'):
-            x = apply_bias_act(dense_layer(x, fmaps=nf(0)), act=act)
-
-    # Output layer with label conditioning from "Which Training Methods for GANs do actually Converge?"
-    with tf.variable_scope('Output'):
-        x = apply_bias_act(dense_layer(x, fmaps=max(labels_in.shape[1], 1)))
-        if labels_in.shape[1] > 0:
-            x = tf.reduce_sum(x * labels_in, axis=1, keepdims=True)
-    scores_out = x
-
-    # Output.
-    assert scores_out.dtype == tf.as_dtype(dtype)
-    scores_out = tf.identity(scores_out, name='scores_out')
-    return scores_out
-
-#----------------------------------------------------------------------------
-# StyleGAN2 discriminator (Figure 7).
-# Implements skip connections and residual nets (Figure 7), but no progressive growing.
-# Used in configs E-F (Table 1).
-
-def D_stylegan2(
-    images_in,                          # First input: Images [minibatch, channel, height, width].
-    labels_in,                          # Second input: Labels [minibatch, label_size].
-    num_channels        = 3,            # Number of input color channels. Overridden based on dataset.
-    resolution          = 1024,         # Input resolution. Overridden based on dataset.
-    label_size          = 0,            # Dimensionality of the labels, 0 if no labels. Overridden based on dataset.
-    fmap_base           = 16 << 10,     # Overall multiplier for the number of feature maps.
-    fmap_decay          = 1.0,          # log2 feature map reduction when doubling the resolution.
-    fmap_min            = 1,            # Minimum number of feature maps in any layer.
-    fmap_max            = 512,          # Maximum number of feature maps in any layer.
-    architecture        = 'resnet',     # Architecture: 'orig', 'skip', 'resnet'.
-    nonlinearity        = 'lrelu',      # Activation function: 'relu', 'lrelu', etc.
-    mbstd_group_size    = 4,            # Group size for the minibatch standard deviation layer, 0 = disable.
-    mbstd_num_features  = 1,            # Number of features for the minibatch standard deviation layer.
-    dtype               = 'float32',    # Data type to use for activations and outputs.
-    resample_kernel     = [1,3,3,1],    # Low-pass filter to apply when resampling activations. None = no filtering.
-    **_kwargs):                         # Ignore unrecognized keyword args.
-
-    resolution_log2 = int(np.log2(resolution))
-    assert resolution == 2**resolution_log2 and resolution >= 4
-    def nf(stage): return np.clip(int(fmap_base / (2.0 ** (stage * fmap_decay))), fmap_min, fmap_max)
-    assert architecture in ['orig', 'skip', 'resnet']
-    act = nonlinearity
-
-    images_in.set_shape([None, num_channels, resolution, resolution])
-    labels_in.set_shape([None, label_size])
-    images_in = tf.cast(images_in, dtype)
-    labels_in = tf.cast(labels_in, dtype)
-
-    # Building blocks for main layers.
-    def fromrgb(x, y, res): # res = 2..resolution_log2
-        with tf.variable_scope('FromRGB'):
-            t = apply_bias_act(conv2d_layer(y, fmaps=nf(res-1), kernel=1), act=act)
-            return t if x is None else x + t
-    def block(x, res): # res = 2..resolution_log2
-        t = x
-        with tf.variable_scope('Conv0'):
-            x = apply_bias_act(conv2d_layer(x, fmaps=nf(res-1), kernel=3), act=act)
-        with tf.variable_scope('Conv1_down'):
-            x = apply_bias_act(conv2d_layer(x, fmaps=nf(res-2), kernel=3, down=True, resample_kernel=resample_kernel), act=act)
-        if architecture == 'resnet':
-            with tf.variable_scope('Skip'):
-                t = conv2d_layer(t, fmaps=nf(res-2), kernel=1, down=True, resample_kernel=resample_kernel)
-                x = (x + t) * (1 / np.sqrt(2))
-        return x
-    def downsample(y):
-        with tf.variable_scope('Downsample'):
-            return downsample_2d(y, k=resample_kernel)
-
-    # Main layers.
-    x = None
-    y = images_in
-    for res in range(resolution_log2, 2, -1):
-        with tf.variable_scope('%dx%d' % (2**res, 2**res)):
-            if architecture == 'skip' or res == resolution_log2:
-                x = fromrgb(x, y, res)
-            x = block(x, res)
-            if architecture == 'skip':
-                y = downsample(y)
-
-    # Final layers.
-    with tf.variable_scope('4x4'):
-        if architecture == 'skip':
-            x = fromrgb(x, y, 2)
-        if mbstd_group_size > 1:
-            with tf.variable_scope('MinibatchStddev'):
-                x = minibatch_stddev_layer(x, mbstd_group_size, mbstd_num_features)
-        with tf.variable_scope('Conv'):
-            x = apply_bias_act(conv2d_layer(x, fmaps=nf(1), kernel=3), act=act)
-        with tf.variable_scope('Dense0'):
-            x = apply_bias_act(dense_layer(x, fmaps=nf(0)), act=act)
-
-    # Output layer with label conditioning from "Which Training Methods for GANs do actually Converge?"
-    with tf.variable_scope('Output'):
-        x = apply_bias_act(dense_layer(x, fmaps=max(labels_in.shape[1], 1)))
-        if labels_in.shape[1] > 0:
-            x = tf.reduce_sum(x * labels_in, axis=1, keepdims=True)
-    scores_out = x
-
-    # Output.
-    assert scores_out.dtype == tf.as_dtype(dtype)
-    scores_out = tf.identity(scores_out, name='scores_out')
-    return scores_out
-
-#----------------------------------------------------------------------------

training/training_loop.py

@@ -1,356 +0,0 @@
-# Copyright (c) 2019, NVIDIA Corporation. All rights reserved.
-#
-# This work is made available under the Nvidia Source Code License-NC.
-# To view a copy of this license, visit
-# https://nvlabs.github.io/stylegan2/license.html
-
-"""Main training script."""
-
-import numpy as np
-import tensorflow as tf
-import dnnlib
-import dnnlib.tflib as tflib
-from dnnlib.tflib.autosummary import autosummary
-
-from training import dataset
-from training import misc
-from metrics import metric_base
-
-#----------------------------------------------------------------------------
-# Just-in-time processing of training images before feeding them to the networks.
-
-def process_reals(x, labels, lod, mirror_augment, drange_data, drange_net):
-    with tf.name_scope('DynamicRange'):
-        x = tf.cast(x, tf.float32)
-        x = misc.adjust_dynamic_range(x, drange_data, drange_net)
-    if mirror_augment:
-        with tf.name_scope('MirrorAugment'):
-            x = tf.where(tf.random_uniform([tf.shape(x)[0]]) < 0.5, x, tf.reverse(x, [3]))
-    with tf.name_scope('FadeLOD'): # Smooth crossfade between consecutive levels-of-detail.
-        s = tf.shape(x)
-        y = tf.reshape(x, [-1, s[1], s[2]//2, 2, s[3]//2, 2])
-        y = tf.reduce_mean(y, axis=[3, 5], keepdims=True)
-        y = tf.tile(y, [1, 1, 1, 2, 1, 2])
-        y = tf.reshape(y, [-1, s[1], s[2], s[3]])
-        x = tflib.lerp(x, y, lod - tf.floor(lod))
-    with tf.name_scope('UpscaleLOD'): # Upscale to match the expected input/output size of the networks.
-        s = tf.shape(x)
-        factor = tf.cast(2 ** tf.floor(lod), tf.int32)
-        x = tf.reshape(x, [-1, s[1], s[2], 1, s[3], 1])
-        x = tf.tile(x, [1, 1, 1, factor, 1, factor])
-        x = tf.reshape(x, [-1, s[1], s[2] * factor, s[3] * factor])
-    return x, labels
-
-#----------------------------------------------------------------------------
-# Evaluate time-varying training parameters.
-
-def training_schedule(
-    cur_nimg,
-    training_set,
-    lod_initial_resolution  = None,     # Image resolution used at the beginning.
-    lod_training_kimg       = 600,      # Thousands of real images to show before doubling the resolution.
-    lod_transition_kimg     = 600,      # Thousands of real images to show when fading in new layers.
-    minibatch_size_base     = 32,       # Global minibatch size.
-    minibatch_size_dict     = {},       # Resolution-specific overrides.
-    minibatch_gpu_base      = 4,        # Number of samples processed at a time by one GPU.
-    minibatch_gpu_dict      = {},       # Resolution-specific overrides.
-    G_lrate_base            = 0.002,    # Learning rate for the generator.
-    G_lrate_dict            = {},       # Resolution-specific overrides.
-    D_lrate_base            = 0.002,    # Learning rate for the discriminator.
-    D_lrate_dict            = {},       # Resolution-specific overrides.
-    lrate_rampup_kimg       = 0,        # Duration of learning rate ramp-up.
-    tick_kimg_base          = 4,        # Default interval of progress snapshots.
-    tick_kimg_dict          = {8:28, 16:24, 32:20, 64:16, 128:12, 256:8, 512:6, 1024:4}): # Resolution-specific overrides.
-
-    # Initialize result dict.
-    s = dnnlib.EasyDict()
-    s.kimg = cur_nimg / 1000.0
-
-    # Training phase.
-    phase_dur = lod_training_kimg + lod_transition_kimg
-    phase_idx = int(np.floor(s.kimg / phase_dur)) if phase_dur > 0 else 0
-    phase_kimg = s.kimg - phase_idx * phase_dur
-
-    # Level-of-detail and resolution.
-    if lod_initial_resolution is None:
-        s.lod = 0.0
-    else:
-        s.lod = training_set.resolution_log2
-        s.lod -= np.floor(np.log2(lod_initial_resolution))
-        s.lod -= phase_idx
-        if lod_transition_kimg > 0:
-            s.lod -= max(phase_kimg - lod_training_kimg, 0.0) / lod_transition_kimg
-        s.lod = max(s.lod, 0.0)
-    s.resolution = 2 ** (training_set.resolution_log2 - int(np.floor(s.lod)))
-
-    # Minibatch size.
-    s.minibatch_size = minibatch_size_dict.get(s.resolution, minibatch_size_base)
-    s.minibatch_gpu = minibatch_gpu_dict.get(s.resolution, minibatch_gpu_base)
-
-    # Learning rate.
-    s.G_lrate = G_lrate_dict.get(s.resolution, G_lrate_base)
-    s.D_lrate = D_lrate_dict.get(s.resolution, D_lrate_base)
-    if lrate_rampup_kimg > 0:
-        rampup = min(s.kimg / lrate_rampup_kimg, 1.0)
-        s.G_lrate *= rampup
-        s.D_lrate *= rampup
-
-    # Other parameters.
-    s.tick_kimg = tick_kimg_dict.get(s.resolution, tick_kimg_base)
-    return s
-
-#----------------------------------------------------------------------------
-# Main training script.
-
-def training_loop(
-    G_args                  = {},       # Options for generator network.
-    D_args                  = {},       # Options for discriminator network.
-    G_opt_args              = {},       # Options for generator optimizer.
-    D_opt_args              = {},       # Options for discriminator optimizer.
-    G_loss_args             = {},       # Options for generator loss.
-    D_loss_args             = {},       # Options for discriminator loss.
-    dataset_args            = {},       # Options for dataset.load_dataset().
-    sched_args              = {},       # Options for train.TrainingSchedule.
-    grid_args               = {},       # Options for train.setup_snapshot_image_grid().
-    metric_arg_list         = [],       # Options for MetricGroup.
-    tf_config               = {},       # Options for tflib.init_tf().
-    data_dir                = None,     # Directory to load datasets from.
-    G_smoothing_kimg        = 10.0,     # Half-life of the running average of generator weights.
-    minibatch_repeats       = 4,        # Number of minibatches to run before adjusting training parameters.
-    lazy_regularization     = True,     # Perform regularization as a separate training step?
-    G_reg_interval          = 4,        # How often the perform regularization for G? Ignored if lazy_regularization=False.
-    D_reg_interval          = 16,       # How often the perform regularization for D? Ignored if lazy_regularization=False.
-    reset_opt_for_new_lod   = True,     # Reset optimizer internal state (e.g. Adam moments) when new layers are introduced?
-    total_kimg              = 25000,    # Total length of the training, measured in thousands of real images.
-    mirror_augment          = False,    # Enable mirror augment?
-    drange_net              = [-1,1],   # Dynamic range used when feeding image data to the networks.
-    image_snapshot_ticks    = 50,       # How often to save image snapshots? None = only save 'reals.png' and 'fakes-init.png'.
-    network_snapshot_ticks  = 50,       # How often to save network snapshots? None = only save 'networks-final.pkl'.
-    save_tf_graph           = False,    # Include full TensorFlow computation graph in the tfevents file?
-    save_weight_histograms  = False,    # Include weight histograms in the tfevents file?
-    resume_pkl              = None,     # Network pickle to resume training from, None = train from scratch.
-    resume_kimg             = 0.0,      # Assumed training progress at the beginning. Affects reporting and training schedule.
-    resume_time             = 0.0,      # Assumed wallclock time at the beginning. Affects reporting.
-    resume_with_new_nets    = False):   # Construct new networks according to G_args and D_args before resuming training?
-
-    # Initialize dnnlib and TensorFlow.
-    tflib.init_tf(tf_config)
-    num_gpus = dnnlib.submit_config.num_gpus
-
-    # Load training set.
-    training_set = dataset.load_dataset(data_dir=dnnlib.convert_path(data_dir), verbose=True, **dataset_args)
-    grid_size, grid_reals, grid_labels = misc.setup_snapshot_image_grid(training_set, **grid_args)
-    misc.save_image_grid(grid_reals, dnnlib.make_run_dir_path('reals.png'), drange=training_set.dynamic_range, grid_size=grid_size)
-
-    # Construct or load networks.
-    with tf.device('/gpu:0'):
-        if resume_pkl is None or resume_with_new_nets:
-            print('Constructing networks...')
-            G = tflib.Network('G', num_channels=training_set.shape[0], resolution=training_set.shape[1], label_size=training_set.label_size, **G_args)
-            D = tflib.Network('D', num_channels=training_set.shape[0], resolution=training_set.shape[1], label_size=training_set.label_size, **D_args)
-            Gs = G.clone('Gs')
-        if resume_pkl is not None:
-            print('Loading networks from "%s"...' % resume_pkl)
-            rG, rD, rGs = misc.load_pkl(resume_pkl)
-            if resume_with_new_nets: G.copy_vars_from(rG); D.copy_vars_from(rD); Gs.copy_vars_from(rGs)
-            else: G = rG; D = rD; Gs = rGs
-
-    # Print layers and generate initial image snapshot.
-    G.print_layers(); D.print_layers()
-    sched = training_schedule(cur_nimg=total_kimg*1000, training_set=training_set, **sched_args)
-    grid_latents = np.random.randn(np.prod(grid_size), *G.input_shape[1:])
-    grid_fakes = Gs.run(grid_latents, grid_labels, is_validation=True, minibatch_size=sched.minibatch_gpu)
-    misc.save_image_grid(grid_fakes, dnnlib.make_run_dir_path('fakes_init.png'), drange=drange_net, grid_size=grid_size)
-
-    # Setup training inputs.
-    print('Building TensorFlow graph...')
-    with tf.name_scope('Inputs'), tf.device('/cpu:0'):
-        lod_in               = tf.placeholder(tf.float32, name='lod_in', shape=[])
-        lrate_in             = tf.placeholder(tf.float32, name='lrate_in', shape=[])
-        minibatch_size_in    = tf.placeholder(tf.int32, name='minibatch_size_in', shape=[])
-        minibatch_gpu_in     = tf.placeholder(tf.int32, name='minibatch_gpu_in', shape=[])
-        minibatch_multiplier = minibatch_size_in // (minibatch_gpu_in * num_gpus)
-        Gs_beta              = 0.5 ** tf.div(tf.cast(minibatch_size_in, tf.float32), G_smoothing_kimg * 1000.0) if G_smoothing_kimg > 0.0 else 0.0
-
-    # Setup optimizers.
-    G_opt_args = dict(G_opt_args)
-    D_opt_args = dict(D_opt_args)
-    for args, reg_interval in [(G_opt_args, G_reg_interval), (D_opt_args, D_reg_interval)]:
-        args['minibatch_multiplier'] = minibatch_multiplier
-        args['learning_rate'] = lrate_in
-        if lazy_regularization:
-            mb_ratio = reg_interval / (reg_interval + 1)
-            args['learning_rate'] *= mb_ratio
-            if 'beta1' in args: args['beta1'] **= mb_ratio
-            if 'beta2' in args: args['beta2'] **= mb_ratio
-    G_opt = tflib.Optimizer(name='TrainG', **G_opt_args)
-    D_opt = tflib.Optimizer(name='TrainD', **D_opt_args)
-    G_reg_opt = tflib.Optimizer(name='RegG', share=G_opt, **G_opt_args)
-    D_reg_opt = tflib.Optimizer(name='RegD', share=D_opt, **D_opt_args)
-
-    # Build training graph for each GPU.
-    data_fetch_ops = []
-    for gpu in range(num_gpus):
-        with tf.name_scope('GPU%d' % gpu), tf.device('/gpu:%d' % gpu):
-
-            # Create GPU-specific shadow copies of G and D.
-            G_gpu = G if gpu == 0 else G.clone(G.name + '_shadow')
-            D_gpu = D if gpu == 0 else D.clone(D.name + '_shadow')
-
-            # Fetch training data via temporary variables.
-            with tf.name_scope('DataFetch'):
-                sched = training_schedule(cur_nimg=int(resume_kimg*1000), training_set=training_set, **sched_args)
-                reals_var = tf.Variable(name='reals', trainable=False, initial_value=tf.zeros([sched.minibatch_gpu] + training_set.shape))
-                labels_var = tf.Variable(name='labels', trainable=False, initial_value=tf.zeros([sched.minibatch_gpu, training_set.label_size]))
-                reals_write, labels_write = training_set.get_minibatch_tf()
-                reals_write, labels_write = process_reals(reals_write, labels_write, lod_in, mirror_augment, training_set.dynamic_range, drange_net)
-                reals_write = tf.concat([reals_write, reals_var[minibatch_gpu_in:]], axis=0)
-                labels_write = tf.concat([labels_write, labels_var[minibatch_gpu_in:]], axis=0)
-                data_fetch_ops += [tf.assign(reals_var, reals_write)]
-                data_fetch_ops += [tf.assign(labels_var, labels_write)]
-                reals_read = reals_var[:minibatch_gpu_in]
-                labels_read = labels_var[:minibatch_gpu_in]
-
-            # Evaluate loss functions.
-            lod_assign_ops = []
-            if 'lod' in G_gpu.vars: lod_assign_ops += [tf.assign(G_gpu.vars['lod'], lod_in)]
-            if 'lod' in D_gpu.vars: lod_assign_ops += [tf.assign(D_gpu.vars['lod'], lod_in)]
-            with tf.control_dependencies(lod_assign_ops):
-                with tf.name_scope('G_loss'):
-                    G_loss, G_reg = dnnlib.util.call_func_by_name(G=G_gpu, D=D_gpu, opt=G_opt, training_set=training_set, minibatch_size=minibatch_gpu_in, **G_loss_args)
-                with tf.name_scope('D_loss'):
-                    D_loss, D_reg = dnnlib.util.call_func_by_name(G=G_gpu, D=D_gpu, opt=D_opt, training_set=training_set, minibatch_size=minibatch_gpu_in, reals=reals_read, labels=labels_read, **D_loss_args)
-
-            # Register gradients.
-            if not lazy_regularization:
-                if G_reg is not None: G_loss += G_reg
-                if D_reg is not None: D_loss += D_reg
-            else:
-                if G_reg is not None: G_reg_opt.register_gradients(tf.reduce_mean(G_reg * G_reg_interval), G_gpu.trainables)
-                if D_reg is not None: D_reg_opt.register_gradients(tf.reduce_mean(D_reg * D_reg_interval), D_gpu.trainables)
-            G_opt.register_gradients(tf.reduce_mean(G_loss), G_gpu.trainables)
-            D_opt.register_gradients(tf.reduce_mean(D_loss), D_gpu.trainables)
-
-    # Setup training ops.
-    data_fetch_op = tf.group(*data_fetch_ops)
-    G_train_op = G_opt.apply_updates()
-    D_train_op = D_opt.apply_updates()
-    G_reg_op = G_reg_opt.apply_updates(allow_no_op=True)
-    D_reg_op = D_reg_opt.apply_updates(allow_no_op=True)
-    Gs_update_op = Gs.setup_as_moving_average_of(G, beta=Gs_beta)
-
-    # Finalize graph.
-    with tf.device('/gpu:0'):
-        try:
-            peak_gpu_mem_op = tf.contrib.memory_stats.MaxBytesInUse()
-        except tf.errors.NotFoundError:
-            peak_gpu_mem_op = tf.constant(0)
-    tflib.init_uninitialized_vars()
-
-    print('Initializing logs...')
-    summary_log = tf.summary.FileWriter(dnnlib.make_run_dir_path())
-    if save_tf_graph:
-        summary_log.add_graph(tf.get_default_graph())
-    if save_weight_histograms:
-        G.setup_weight_histograms(); D.setup_weight_histograms()
-    metrics = metric_base.MetricGroup(metric_arg_list)
-
-    print('Training for %d kimg...\n' % total_kimg)
-    dnnlib.RunContext.get().update('', cur_epoch=resume_kimg, max_epoch=total_kimg)
-    maintenance_time = dnnlib.RunContext.get().get_last_update_interval()
-    cur_nimg = int(resume_kimg * 1000)
-    cur_tick = -1
-    tick_start_nimg = cur_nimg
-    prev_lod = -1.0
-    running_mb_counter = 0
-    while cur_nimg < total_kimg * 1000:
-        if dnnlib.RunContext.get().should_stop(): break
-
-        # Choose training parameters and configure training ops.
-        sched = training_schedule(cur_nimg=cur_nimg, training_set=training_set, **sched_args)
-        assert sched.minibatch_size % (sched.minibatch_gpu * num_gpus) == 0
-        training_set.configure(sched.minibatch_gpu, sched.lod)
-        if reset_opt_for_new_lod:
-            if np.floor(sched.lod) != np.floor(prev_lod) or np.ceil(sched.lod) != np.ceil(prev_lod):
-                G_opt.reset_optimizer_state(); D_opt.reset_optimizer_state()
-        prev_lod = sched.lod
-
-        # Run training ops.
-        feed_dict = {lod_in: sched.lod, lrate_in: sched.G_lrate, minibatch_size_in: sched.minibatch_size, minibatch_gpu_in: sched.minibatch_gpu}
-        for _repeat in range(minibatch_repeats):
-            rounds = range(0, sched.minibatch_size, sched.minibatch_gpu * num_gpus)
-            run_G_reg = (lazy_regularization and running_mb_counter % G_reg_interval == 0)
-            run_D_reg = (lazy_regularization and running_mb_counter % D_reg_interval == 0)
-            cur_nimg += sched.minibatch_size
-            running_mb_counter += 1
-
-            # Fast path without gradient accumulation.
-            if len(rounds) == 1:
-                tflib.run([G_train_op, data_fetch_op], feed_dict)
-                if run_G_reg:
-                    tflib.run(G_reg_op, feed_dict)
-                tflib.run([D_train_op, Gs_update_op], feed_dict)
-                if run_D_reg:
-                    tflib.run(D_reg_op, feed_dict)
-
-            # Slow path with gradient accumulation.
-            else:
-                for _round in rounds:
-                    tflib.run(G_train_op, feed_dict)
-                if run_G_reg:
-                    for _round in rounds:
-                        tflib.run(G_reg_op, feed_dict)
-                tflib.run(Gs_update_op, feed_dict)
-                for _round in rounds:
-                    tflib.run(data_fetch_op, feed_dict)
-                    tflib.run(D_train_op, feed_dict)
-                if run_D_reg:
-                    for _round in rounds:
-                        tflib.run(D_reg_op, feed_dict)
-
-        # Perform maintenance tasks once per tick.
-        done = (cur_nimg >= total_kimg * 1000)
-        if cur_tick < 0 or cur_nimg >= tick_start_nimg + sched.tick_kimg * 1000 or done:
-            cur_tick += 1
-            tick_kimg = (cur_nimg - tick_start_nimg) / 1000.0
-            tick_start_nimg = cur_nimg
-            tick_time = dnnlib.RunContext.get().get_time_since_last_update()
-            total_time = dnnlib.RunContext.get().get_time_since_start() + resume_time
-
-            # Report progress.
-            print('tick %-5d kimg %-8.1f lod %-5.2f minibatch %-4d time %-12s sec/tick %-7.1f sec/kimg %-7.2f maintenance %-6.1f gpumem %.1f' % (
-                autosummary('Progress/tick', cur_tick),
-                autosummary('Progress/kimg', cur_nimg / 1000.0),
-                autosummary('Progress/lod', sched.lod),
-                autosummary('Progress/minibatch', sched.minibatch_size),
-                dnnlib.util.format_time(autosummary('Timing/total_sec', total_time)),
-                autosummary('Timing/sec_per_tick', tick_time),
-                autosummary('Timing/sec_per_kimg', tick_time / tick_kimg),
-                autosummary('Timing/maintenance_sec', maintenance_time),
-                autosummary('Resources/peak_gpu_mem_gb', peak_gpu_mem_op.eval() / 2**30)))
-            autosummary('Timing/total_hours', total_time / (60.0 * 60.0))
-            autosummary('Timing/total_days', total_time / (24.0 * 60.0 * 60.0))
-
-            # Save snapshots.
-            if image_snapshot_ticks is not None and (cur_tick % image_snapshot_ticks == 0 or done):
-                grid_fakes = Gs.run(grid_latents, grid_labels, is_validation=True, minibatch_size=sched.minibatch_gpu)
-                misc.save_image_grid(grid_fakes, dnnlib.make_run_dir_path('fakes%06d.png' % (cur_nimg // 1000)), drange=drange_net, grid_size=grid_size)
-            if network_snapshot_ticks is not None and (cur_tick % network_snapshot_ticks == 0 or done):
-                pkl = dnnlib.make_run_dir_path('network-snapshot-%06d.pkl' % (cur_nimg // 1000))
-                misc.save_pkl((G, D, Gs), pkl)
-                metrics.run(pkl, run_dir=dnnlib.make_run_dir_path(), data_dir=dnnlib.convert_path(data_dir), num_gpus=num_gpus, tf_config=tf_config)
-
-            # Update summaries and RunContext.
-            metrics.update_autosummaries()
-            tflib.autosummary.save_summaries(summary_log, cur_nimg)
-            dnnlib.RunContext.get().update('%.2f' % sched.lod, cur_epoch=cur_nimg // 1000, max_epoch=total_kimg)
-            maintenance_time = dnnlib.RunContext.get().get_last_update_interval() - tick_time
-
-    # Save final snapshot.
-    misc.save_pkl((G, D, Gs), dnnlib.make_run_dir_path('network-final.pkl'))
-
-    # All done.
-    summary_log.close()
-    training_set.close()
-
-#----------------------------------------------------------------------------