Instruction:
Given the following Python 3 function, write the documentation
def _enqueue_batches(self, stop_event: Event) -> None:
        while True:
            self._stream = self._get_stream()
            while True:
                # Acquire the semaphore before processing the next batch
                # but immediately release it so that other threads
                # are not blocked when they decide to acquire it again.
                with self._semaphore:
                    pass
                # It always takes a short moment before the native call actually
                # releases the GIL and we are free to compute. The following sleep
                # is here to compensate for this short moment - we don't want to
                # slow down the native call before the GIL is released.
                time.sleep(CXF_BUFFER_SLEEP)
                try:
                    batch = next(self._stream)
                except StopIteration:
                    break
                self._queue.put(batch)
                self._batch_count += 1
                if stop_event.is_set():
                    return
                if self._epoch_limit_reached():
                    self._queue.put(None)
                    self._batch_count = 0
                    return
            self._stream = None  # yield a new iterator next time
            if self._epoch_size <= 0:  # for non-fixed size epochs
                self._queue.put(None)
                self._batch_count = 0
                return

Response: Enqueue all the stream batches.


Instruction:
Can you generate the documentation for the following Python 3 function
def _dequeue_batch(self) -> Optional[Batch]:
        if self._enqueueing_thread is None:
            raise ValueError('StreamWrapper `{}` with buffer of size `{}` was used outside with-resource environment.'
                             .format(self._name, self._buffer_size))
        if not self._enqueueing_thread.is_alive() and self._queue.empty():
            self._start_thread()
        while True:
            try:
                batch = self._queue.get(timeout=2)
                self._queue.task_done()
                break
            except Empty:
                if not self._enqueueing_thread.is_alive():
                    try:
                        # the enqueueing thread may just finished properly so lets check the queue eagerly
                        batch = self._queue.get_nowait()
                        self._queue.task_done()
                        break
                    except Empty:
                        # so we failed to retrieve a batch and the enqueueing thread is dead
                        # there is no hope, something must went wrong
                        raise ChildProcessError('Enqueueing thread ended unexpectedly.')
        return batch

Response: Returns a single batch from the queue or None if the queue is empty.


Instruction:
Make a summary of the following Python 3 code
def _next_batch(self) -> Optional[Batch]:
        if self._epoch_limit_reached():
            self._batch_count = 0
            return None
        try:
            batch = next(self._get_stream())
            self._batch_count += 1
            return batch
        except StopIteration:
            self._stream = None  # yield a new iterator next time
            if self._epoch_size > 0:  # underlying stream ended but our fixed size epoch did not
                batch = next(self._get_stream())  # get another stream and return its 1st batch
                self._batch_count += 1
                return batch
            else:
                self._batch_count = 0
                return None

Response: Returns the next batch or None if there are no more batches.


Instruction:
Given the following Python 3 function, write the documentation
def _start_thread(self):
        self._stopping_event = Event()
        self._enqueueing_thread = Thread(target=self._enqueue_batches, args=(self._stopping_event,))
        self._enqueueing_thread.start()

Response: Start an enqueueing thread.


Instruction:
Explain what the following Python 3 code does
def _stop_thread(self):
        self._stopping_event.set()
        queue_content = []
        try:  # give the enqueueing thread chance to put a batch to the queue and check the stopping event
            while True:
                queue_content.append(self._queue.get_nowait())
        except Empty:
            pass
        self._enqueueing_thread.join()
        try:
            queue_content.append(self._queue.get_nowait())  # collect the very last item
        except Empty:
            pass
        self._queue = Queue(max(len(queue_content), self._buffer_size))  # queue content may be bigger than queue size
        for batch in queue_content:
            self._queue.put(batch)

Response: Stop the enqueueing thread. Keep the queue content and stream state.


Instruction:
Write a Python 3 function that can
save the model every n_epochs epoch.

Response:
def _after_n_epoch(self, epoch_id: int, **_) -> None:
        """
        Save the model every ``n_epochs`` epoch.

        :param epoch_id: number of the processed epoch
        """
        SaveEvery.save_model(model=self._model, name_suffix=str(epoch_id), on_failure=self._on_save_failure)


Instruction:
Given the following Python 3 function, write the documentation
def save_model(model: AbstractModel, name_suffix: str, on_failure: str) -> None:
        try:
            logging.debug('Saving the model')
            save_path = model.save(name_suffix)
            logging.info('Model saved to: %s', save_path)
        except Exception as ex:  # pylint: disable=broad-except
            if on_failure == 'error':
                raise IOError('Failed to save the model.') from ex
            elif on_failure == 'warn':
                logging.warning('Failed to save the model.')

Response: Save the given model with the given name_suffix. On failure take the specified action.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _get_value(self, epoch_data: EpochData) -> float:
        if self._stream_name not in epoch_data:
            raise KeyError('Stream `{}` was not found in the epoch data.\nAvailable streams are `{}`.'
                           .format(self._stream_name, epoch_data.keys()))

        stream_data = epoch_data[self._stream_name]
        if self._variable not in stream_data:
            raise KeyError('Variable `{}` for stream `{}` was not found in the epoch data. '
                           'Available variables for stream `{}` are `{}`.'
                           .format(self._variable, self._stream_name, self._stream_name, stream_data.keys()))

        value = stream_data[self._variable]
        if self._aggregation:
            if not isinstance(value, dict):
                raise TypeError('Variable `{}` is expected to be a dict when aggregation is specified. '
                                'Got `{}` instead.'.format(self._variable, type(value).__name__))
            if self._aggregation not in value:
                raise KeyError('Specified aggregation `{}` was not found in the variable `{}`. '
                               'Available aggregations: `{}`.'.format(self._aggregation, self._variable, value.keys()))
            value = value[self._aggregation]
        if not np.isscalar(value):
            raise ValueError('Variable `{}` value is not a scalar.'.format(value))

        return value

Response: Retrieve the value of the monitored variable from the given epoch data.


Instruction:
Can you generate the documentation for the following Python 3 function
def _is_value_better(self, new_value: float) -> bool:
        if self._best_value is None:
            return True
        if self._condition == 'min':
            return new_value < self._best_value
        if self._condition == 'max':
            return new_value > self._best_value

Response: Test if the new value is better than the best so far.


Instruction:
Explain what the following Python 3 code does
def after_epoch(self, epoch_data: EpochData, **_) -> None:
        new_value = self._get_value(epoch_data)

        if self._is_value_better(new_value):
            self._best_value = new_value
            SaveEvery.save_model(model=self._model, name_suffix=self._OUTPUT_NAME, on_failure=self._on_save_failure)

Response: Save the model if the new value of the monitored variable is better than the best value so far.


Instruction:
Can you implement a function in Python 3 that
prints a progress bar with the given prefix and suffix.

Response:
def print_progress_bar(done: int, total: int, prefix: str = '', suffix: str = '') -> None:
    """
    Print a progressbar with the given prefix and suffix, without newline at the end.

    param done: current step in computation
    param total: total count of steps in computation
    param prefix: info text displayed before the progress bar
    param suffix: info text displayed after the progress bar
    """

    percent = '{0:.1f}'.format(100 * (done / float(total)))
    base_len = shutil.get_terminal_size().columns - 7 - len(prefix) - len(suffix)
    base_len = min([base_len, 50])
    min_length = base_len - 1 - len('{}/{}={}'.format(total, total, '100.0'))
    length = base_len - len('{}/{}={}'.format(done, total, percent))
    if min_length > 0:
        filled_len = int(min_length * done // total)
        bar = '='*filled_len + '-'*(min_length - filled_len)
        spacing = ' '*(length - min_length)
        print('\r{}: |{}|{}{}/{}={}% {}'.format(prefix, bar, spacing, done, total, percent, suffix), end='\r')
    else:
        short_progress = '\r{}: {}/{}'.format(prefix, done, total)
        if len(short_progress) <= shutil.get_terminal_size().columns:
            print(short_progress, end='\r')
        else:
            print(['-', '\\', '|', '/'][done % 4], end='\r')


Instruction:
Can you implement a function in Python 3 that
converts seconds to the time format H M S. UU.

Response:
def get_formatted_time(seconds: float) -> str:
    """
    Convert seconds to the time format ``H:M:S.UU``.

    :param seconds: time in seconds
    :return: formatted human-readable time
    """
    seconds = round(seconds)
    m, s = divmod(seconds, 60)
    h, m = divmod(m, 60)
    return '{:d}:{:02d}:{:02d}'.format(h, m, s)


Instruction:
Write a Python 3 function for
printing the current batch count and progress bar for the current stream.

Response:
def after_batch(self, stream_name: str, batch_data: Batch) -> None:
        """
        Display the progress and ETA for the current stream in the epoch.
        If the stream size (total batch count) is unknown (1st epoch), print only the number of processed batches.
        """
        if self._current_stream_name is None or self._current_stream_name != stream_name:
            self._current_stream_name = stream_name
            self._current_stream_start = None
        erase_line()

        self._current_batch_count[stream_name] += 1
        current_batch = self._current_batch_count[stream_name]

        # total batch count is available
        if stream_name in self._total_batch_count:
            # compute ETA
            total_batches = self._total_batch_count[stream_name]
            if self._current_stream_start:
                measured_batches = current_batch - 1
                avg_batch_time = (time.time() - self._current_stream_start) / measured_batches
                eta_sec = avg_batch_time * (total_batches - current_batch)
                eta = get_formatted_time(eta_sec)
            else:
                self._current_stream_start = time.time()
                eta = ''

            print_progress_bar(current_batch, total_batches, prefix=stream_name, suffix=eta)

        # total batch count is not available (1st epoch)
        else:
            short_progress = '{}: {}'.format(stream_name, current_batch)
            if len(short_progress) <= shutil.get_terminal_size().columns:
                print(short_progress, end='\r')
            else:
                print(['-', '\\', '|', '/'][current_batch % 4],  end='\r')


Instruction:
Explain what the following Python 3 code does
def after_epoch(self, **_) -> None:
        if not self._total_batch_count_saved:
            self._total_batch_count = self._current_batch_count.copy()
            self._total_batch_count_saved = True
        self._current_batch_count.clear()
        self._current_stream_start = None
        self._current_stream_name = None
        erase_line()

Response: Reset the internal state of the internal state of the class.


Instruction:
How would you explain what the following Python 3 function does
def after_epoch_profile(self, epoch_id, profile: TimeProfile, train_stream_name: str, extra_streams: Iterable[str]) -> None:

        read_data_total = 0
        eval_total = 0
        train_total = sum(profile.get('eval_batch_{}'.format(train_stream_name), []))
        hooks_total = sum(profile.get('after_epoch_hooks', []))

        for stream_name in chain(extra_streams, [train_stream_name]):
            read_data_total += sum(profile.get('read_batch_' + stream_name, []))
            hooks_total += sum(profile.get('after_batch_hooks_' + stream_name, []))
            if stream_name != train_stream_name:
                eval_total += sum(profile.get('eval_batch_' + stream_name, []))

        logging.info('\tT read data:\t%f', read_data_total)
        logging.info('\tT train:\t%f', train_total)
        logging.info('\tT eval:\t%f', eval_total)
        logging.info('\tT hooks:\t%f', hooks_total)

Response: Summarize and log the given epoch profile.


Instruction:
Make a summary of the following Python 3 code
def confusion_matrix(expected: np.ndarray, predicted: np.ndarray, num_classes: int) -> np.ndarray:
    assert np.issubclass_(expected.dtype.type, np.integer), " Classes' indices must be integers"
    assert np.issubclass_(predicted.dtype.type, np.integer), " Classes' indices must be integers"
    assert expected.shape == predicted.shape, "Predicted and expected data must be the same length"
    assert num_classes > np.max([predicted, expected]), \
        "Number of classes must be at least the number of indices in predicted/expected data"
    assert np.min([predicted, expected]) >= 0, " Classes' indices must be positive integers"
    cm_abs = np.zeros((num_classes, num_classes), dtype=np.int32)
    for pred, exp in zip(predicted, expected):
        cm_abs[exp, pred] += 1
    return cm_abs

Response: Calculate and return the confusion matrix for the predicted and expected labels.


Instruction:
How would you explain what the following Python 3 function does
def _build_grid_search_commands(script: str, params: typing.Iterable[str]) -> typing.Iterable[typing.List[str]]:

    param_space = OrderedDict()
    for arg in params:
        assert '=' in arg

        name = arg[:arg.index('=')]
        options = arg[arg.index('=') + 1:]
        options = ast.literal_eval(options)
        assert isinstance(options, list), options

        param_space[name] = options

    param_names = param_space.keys()
    commands = []
    for values in itertools.product(*[param_space[name] for name in param_names]):
        command = str(script).split()
        for name, value in zip(param_names, values):
            command.append(str(name) + '="' + str(value) + '"')
        commands.append(command)

    return commands

Response: Builds all grid search commands.


Instruction:
Write a Python 3 script to
build all grid search parameters and optionally run them.

Response:
def grid_search(script: str, params: typing.Iterable[str], dry_run: bool=False) -> None:
    """
    Build all grid search parameter configurations and optionally run them.

    :param script: String of command prefix, e.g. ``cxflow train -v -o log``.
    :param params: Iterable collection of strings in standard **cxflow** param form, e.g. ``'numerical_param=[1, 2]'``
                   or ``'text_param=["hello", "cio"]'``.
    :param dry_run: If set to ``True``, the built commands will only be printed instead of executed.
    """

    commands = _build_grid_search_commands(script=script, params=params)

    if dry_run:
        logging.warning('Dry run')
        for command in commands:
            logging.info(command)
    else:
        for command in commands:
            try:
                completed_process = subprocess.run(command)
                logging.info('Command `%s` completed with exit code %d', command, completed_process.returncode)
            except Exception as _:  # pylint: disable=broad-except
                logging.error('Command `%s` failed.', command)


Instruction:
Can you tell what is the following Python 3 function doing
def stream_info(self) -> None:
        stream_names = [stream_name for stream_name in dir(self)
                        if 'stream' in stream_name and stream_name != 'stream_info']
        logging.info('Found %s stream candidates: %s', len(stream_names), stream_names)
        for stream_name in stream_names:
            try:
                stream_fn = getattr(self, stream_name)
                logging.info(stream_name)
                batch = next(iter(stream_fn()))
                rows = []
                for key, value in batch.items():
                        try:
                            value_arr = np.array(value)
                            row = [key, value_arr.dtype, value_arr.shape]
                            if value_arr.dtype.kind in 'bui':  # boolean, unsigned, integer
                                row.append('{} - {}'.format(value_arr.min(), value_arr.max()))
                            elif value_arr.dtype.kind is 'f':
                                row.append('{0:.2f} - {1:.2f}'.format(value_arr.min(), value_arr.max()))
                        except ValueError:  # np broadcasting failed (ragged array)
                            value_arr = None
                            row = [key, '{}'.format(type(value[0]).__name__), '({},)'.format(len(list(value)))]

                        if value_arr is None or \
                                (value_arr.ndim > 0 and value_arr.shape[1:] != np.array(value_arr[0]).shape):
                            logging.warning('*** stream source `%s` appears to be ragged (non-rectangular) ***', key)

                        rows.append(row)
                for line in tabulate.tabulate(rows, headers=['name', 'dtype', 'shape', 'range'],
                                              tablefmt='grid').split('\n'):
                    logging.info(line)
            except Exception:
                logging.warning('Exception was raised during checking stream `%s`, '
                                '(stack trace is displayed only with --verbose flag)', stream_name)
                logging.debug(traceback.format_exc())

Response: Check and report source names dtypes and shapes of all the streams available.


Instruction:
Can you generate the documentation for the following Python 3 function
def parse_arg(arg: str) -> typing.Tuple[str, typing.Any]:
    assert '=' in arg, 'Unrecognized argument `{}`. [name]=[value] expected.'.format(arg)

    key = arg[:arg.index('=')]
    value = yaml.load(arg[arg.index('=') + 1:])

    return key, value

Response: Parse a string in format key = value to tuple


Instruction:
Here you have a function in Python 3, explain what it does
def load_config(config_file: str, additional_args: typing.Iterable[str]=()) -> dict:

    config = load_yaml(config_file)

    for key_full, value in [parse_arg(arg) for arg in additional_args]:
        key_split = key_full.split('.')
        key_prefix = key_split[:-1]
        key = key_split[-1]

        conf = config
        for key_part in key_prefix:
            conf = conf[key_part]
        conf[key] = value

    return reload(config)

Response: Load config from YAML config_file and extend or override it with the given additional_args.


Instruction:
Given the following Python 3 function, write the documentation
def find_config(config_path: str) -> str:
    if path.isdir(config_path):  # dir specified instead of config file
        config_path = path.join(config_path, CXF_CONFIG_FILE)
    assert path.exists(config_path), '`{}` does not exist'.format(config_path)
    return config_path

Response: Derive configuration file path from the given path and check its existence.


Instruction:
Can you generate the documentation for the following Python 3 function
def fallback(message: str, ex: Exception) -> None:
    logging.error('%s', message)
    logging.exception('%s', ex)
    sys.exit(1)

Response: Fallback procedure when a cli command fails.


Instruction:
Here you have a function in Python 3, explain what it does
def _configure_dataset(self, data_root: str=None, download_urls: Iterable[str]=None, **kwargs) -> None:
        self._data_root = data_root
        self._download_urls = download_urls

Response: Configure the internal properties of the object.


Instruction:
Can you generate the documentation for the following Python 3 function
def resume(config_path: str, restore_from: Optional[str], cl_arguments: Iterable[str], output_root: str) -> None:
    config = None

    try:
        config_path = find_config(config_path)
        restore_from = restore_from or path.dirname(config_path)
        config = load_config(config_file=config_path, additional_args=cl_arguments)

        validate_config(config)

        logging.debug('\tLoaded config: %s', config)
    except Exception as ex:  # pylint: disable=broad-except
        fallback('Loading config failed', ex)

    run(config=config, output_root=output_root, restore_from=restore_from)

Response: Resume training from the specified directory and start training.


Instruction:
Can you tell what is the following Python 3 function doing
def after_epoch_profile(self, epoch_id: int, profile: TimeProfile, train_stream_name: str, extra_streams: Iterable[str]) -> None:
        pass

Response: This event provides opportunity to process time profile of the finished epoch.


Instruction:
Can you write a function in Python 3 where it
loads the YAML file.

Response:
def load_yaml(yaml_file: str) -> Any:
    """
    Load YAML from file.

    :param yaml_file: path to YAML file
    :return: content of the YAML as dict/list
    """
    with open(yaml_file, 'r') as file:
        return ruamel.yaml.load(file, ruamel.yaml.RoundTripLoader)


Instruction:
Write a Python 3 script to
save the given dict to a file in YAML format.

Response:
def yaml_to_file(data: Mapping, output_dir: str, name: str) -> str:
    """
    Save the given object to the given path in YAML.

    :param data: dict/list to be dumped
    :param output_dir: target output directory
    :param name: target filename
    :return: target path
    """
    dumped_config_f = path.join(output_dir, name)
    with open(dumped_config_f, 'w') as file:
        yaml.dump(data, file, Dumper=ruamel.yaml.RoundTripDumper)
    return dumped_config_f


Instruction:
How would you explain what the following Python 3 function does
def yaml_to_str(data: Mapping) -> str:
    return yaml.dump(data, Dumper=ruamel.yaml.RoundTripDumper)

Response: Return the given configuration as YAML str.


Instruction:
Make a summary of the following Python 3 code
def make_simple(data: Any) -> Any:
    return yaml.load(yaml.dump(data, Dumper=ruamel.yaml.RoundTripDumper), Loader=ruamel.yaml.Loader)

Response: Make a new node with simple data.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def reload(data: Any) -> Any:
    return yaml.load(yaml.dump(data, Dumper=ruamel.yaml.RoundTripDumper), Loader=ruamel.yaml.RoundTripLoader)

Response: Dump and load yaml data.


Instruction:
Write a Python 3 function that can
call after the epoch has been completed.

Response:
def after_epoch(self, epoch_id: int, epoch_data: EpochData) -> None:
        """
        Call :py:meth:`_on_plateau_action` if the ``long_term``
        variable mean is lower/greater than the ``short_term`` mean.
        """

        super().after_epoch(epoch_id=epoch_id, epoch_data=epoch_data)

        self._saved_loss.append(epoch_data[self._stream][self._variable][OnPlateau._AGGREGATION])

        long_mean = np.mean(self._saved_loss[-self._long_term:])
        short_mean = np.mean(self._saved_loss[-self._short_term:])
        if self._objective == 'min' and long_mean < short_mean:
            self._on_plateau_action(epoch_id=epoch_id, epoch_data=epoch_data)
        elif self._objective == 'max' and long_mean > short_mean:
            self._on_plateau_action(epoch_id=epoch_id, epoch_data=epoch_data)


Instruction:
Can you write a function in Python 3 where it
raises an exception when some of the monitored data is NaN.

Response:
def _check_nan(self, epoch_data: EpochData) -> None:
        """
        Raise an exception when some of the monitored data is NaN.

        :param epoch_data: epoch data checked
        :raise KeyError: if the specified variable is not found in the stream
        :raise ValueError: if the variable value is of unsupported type and ``self._on_unknown_type`` is set to ``error``
        """
        for stream_name in epoch_data.keys():
            stream_data = epoch_data[stream_name]
            variables = self._variables if self._variables is not None else stream_data.keys()
            for variable in variables:
                if variable not in stream_data:
                    raise KeyError('Variable `{}` to be nan-checked was not found in the batch data for stream `{}`. '
                                   'Available variables are `{}`.'.format(variable, stream_name, stream_data.keys()))

                value = stream_data[variable]
                if self._is_nan(variable, value):
                    raise TrainingTerminated('Variable `{}` is NaN.'.format(variable))


Instruction:
Write a Python 3 script for
stopping training once the epoch data contains a monitored variable equal to NaN.

Response:
def after_epoch(self, epoch_data: EpochData, **kwargs) -> None:
        """
        If initialized to check after each epoch, stop the training once the epoch data contains a monitored
        variable equal to NaN.

        :param epoch_data: epoch data to be checked
        """

        if self._after_epoch:
            self._check_nan(epoch_data)


Instruction:
Implement a Python 3 function for
stopping training once the batch data contains a monitored variable equal to NaN.

Response:
def after_batch(self, stream_name: str, batch_data) -> None:
        """
        If initialized to check after each batch, stop the training once the batch data contains a monitored
        variable equal to NaN.

        :param stream_name: name of the stream to be checked
        :param batch_data: batch data to be checked
        """

        if self._after_batch:
            self._check_nan({stream_name: batch_data})


Instruction:
Write a Python 3 function for
calling _after_n_epochs method every n_epochs epoch.

Response:
def after_epoch(self, epoch_id: int, **kwargs) -> None:
        """
        Call ``_after_n_epoch`` method every ``n_epochs`` epoch.

        :param epoch_id: number of the processed epoch
        """
        if epoch_id % self._n_epochs == 0:
            self._after_n_epoch(epoch_id=epoch_id, **kwargs)


Instruction:
Can you generate the documentation for the following Python 3 function
def path_total_size(path_: str) -> int:
    if path.isfile(path_):
        return path.getsize(path_)
    total_size = 0
    for root_dir, _, files in os.walk(path_):
        for file_ in files:
            total_size += path.getsize(path.join(root_dir, file_))
    return total_size

Response: Compute total size of the given file or directory.


Instruction:
Can you create a Python 3 function that
returns human readable size and unit from the given filesize in bytes.

Response:
def humanize_filesize(filesize: int) -> Tuple[str, str]:
    """Return human readable pair of size and unit from the given filesize in bytes."""
    for unit in ['', 'K', 'M', 'G', 'T', 'P', 'E', 'Z']:
        if filesize < 1024.0:
            return '{:3.1f}'.format(filesize), unit+'B'
        filesize /= 1024.0


Instruction:
Can you tell what is the following Python 3 function doing
def is_train_dir(dir_: str) -> bool:
    return path.exists(path.join(dir_, CXF_CONFIG_FILE)) and \
           path.exists(path.join(dir_, CXF_TRACE_FILE)) and \
           path.exists(path.join(dir_, CXF_LOG_FILE))

Response: Test if the given dir contains training artifacts.


Instruction:
Can you write a function in Python 3 where it
walks the train dirs and returns a generator of tuples.

Response:
def walk_train_dirs(root_dir: str) -> Iterable[Tuple[str, Iterable[str]]]:
    """
    Modify os.walk with the following:
        - return only root_dir and sub-dirs
        - return only training sub-dirs
        - stop recursion at training dirs

    :param root_dir: root dir to be walked
    :return: generator of (root_dir, training sub-dirs) pairs
    """
    if is_train_dir(root_dir):
        yield '', [root_dir]
        return
    for dir_, subdirs, _ in os.walk(root_dir, topdown=True):
        # filter train sub-dirs
        train_subdirs = [subdir for subdir in subdirs if is_train_dir(path.join(dir_, subdir))]

        # stop the recursion at the train sub-dirs
        for subdir in train_subdirs:
            subdirs.remove(subdir)

        yield dir_, train_subdirs


Instruction:
Explain what the following Python 3 code does
def _print_trainings_long(trainings: Iterable[Tuple[str, dict, TrainingTrace]]) -> None:
    long_table = []
    for train_dir, config, trace in trainings:
        start_datetime, end_datetime = trace[TrainingTraceKeys.TRAIN_BEGIN], trace[TrainingTraceKeys.TRAIN_END]
        if start_datetime:
            age = format_timedelta(datetime.now() - start_datetime) + ' ago'
            if end_datetime:
                duration = format_timedelta(end_datetime - start_datetime)
            else:
                duration = CXF_NA_STR
        else:
            age = CXF_NA_STR
            duration = CXF_NA_STR

        epochs_done = trace[TrainingTraceKeys.EPOCHS_DONE] if trace[TrainingTraceKeys.EPOCHS_DONE] else 0

        long_table.append([path.basename(train_dir)] +
                          list(map(lambda fq_name: fq_name.split('.')[-1], get_classes(config))) +
                          [age, duration, epochs_done])

    print(tabulate(long_table, tablefmt='plain'))

Response: Print a plain table with the details of the given trainings.


Instruction:
Create a Python 3 function for
printing names of the train dirs contained in the given dir.

Response:
def _ls_print_listing(dir_: str, recursive: bool, all_: bool, long: bool) -> List[Tuple[str, dict, TrainingTrace]]:
    """
    Print names of the train dirs contained in the given dir.

    :param dir_: dir to be listed
    :param recursive: walk recursively in sub-directories, stop at train dirs (--recursive option)
    :param all_: include train dirs with no epochs done (--all option)
    :param long: list more details including model name, model and dataset classes,
                 age, duration and epochs done (--long option)
    :return: list of found training tuples (train_dir, configuration dict, trace)
    """
    all_trainings = []
    for root_dir, train_dirs in walk_train_dirs(dir_):
        if train_dirs:
            if recursive:
                print(root_dir + ':')
            trainings = [(train_dir,
                          load_config(path.join(train_dir, CXF_CONFIG_FILE), []),
                          TrainingTrace.from_file(path.join(train_dir, CXF_TRACE_FILE)))
                         for train_dir
                         in [os.path.join(root_dir, train_dir) for train_dir in train_dirs]]
            if not all_:
                trainings = [train_dir for train_dir in trainings if train_dir[2][TrainingTraceKeys.EPOCHS_DONE]]
            if long:
                print('total {}'.format(len(trainings)))
                _print_trainings_long(trainings)
            else:
                for train_dir, _, _ in trainings:
                    print(path.basename(train_dir))
            all_trainings.extend(trainings)
            if recursive:
                print()

        if not recursive:
            break
    return all_trainings


Instruction:
Create a Python 3 function for
printing trainings summary. In particular print tables summarizing the number of trainings with - particular model names - particular combinations of models and datasets :param all_trainings: a list of training tuples (train_dir, configuration dict, trace)

Response:
def _ls_print_summary(all_trainings: List[Tuple[str, dict, TrainingTrace]]) -> None:
    """
    Print trainings summary.
    In particular print tables summarizing the number of trainings with
        - particular model names
        - particular combinations of models and datasets

    :param all_trainings: a list of training tuples (train_dir, configuration dict, trace)
    """
    counts_by_name = defaultdict(int)
    counts_by_classes = defaultdict(int)
    for _, config, _ in all_trainings:
        counts_by_name[get_model_name(config)] += 1
        counts_by_classes[get_classes(config)] += 1

    print_boxed('summary')
    print()

    counts_table = [[name, count] for name, count in counts_by_name.items()]
    print(tabulate(counts_table, headers=['model.name', 'count'], tablefmt='grid'))
    print()

    counts_table = [[classes[0], classes[1], count] for classes, count in counts_by_classes.items()]
    print(tabulate(counts_table, headers=['model.class', 'dataset.class', 'count'], tablefmt='grid'))
    print()


Instruction:
Make a summary of the following Python 3 code
def _ls_print_verbose(training: Tuple[str, dict, str]) -> None:
    train_dir, config, _ = training
    print_boxed('config')
    print(yaml_to_str(config))
    print()

    print_boxed('artifacts')
    _, dirs, files = next(os.walk(train_dir))
    artifacts = [('d', dir) for dir in dirs] + \
                [('-', file_) for file_ in files if file_ not in [CXF_CONFIG_FILE, CXF_LOG_FILE, CXF_TRACE_FILE]]
    artifacts = [(type_, name) + humanize_filesize(path_total_size(path.join(train_dir, name)))
                 for type_, name in artifacts]
    print(tabulate(artifacts, tablefmt='plain', floatfmt='3.1f'))
    print()

Response: Print config and artifacts info from the given training tuple.


Instruction:
Can you create a Python 3 function that
lists training dirs contained in the given dir.

Response:
def list_train_dirs(dir_: str, recursive: bool, all_: bool, long: bool, verbose: bool) -> None:
    """
    List training dirs contained in the given dir with options and outputs similar to the regular `ls` command.
    The function is accessible through cxflow CLI `cxflow ls`.

    :param dir_: dir to be listed
    :param recursive: walk recursively in sub-directories, stop at train dirs (--recursive option)
    :param all_: include train dirs with no epochs done (--all option)
    :param long: list more details including model name, odel and dataset class,
                 age, duration and epochs done (--long option)
    :param verbose: print more verbose output with list of additional artifacts and training config,
                    applicable only when a single train dir is listed (--verbose option)
    """
    if verbose:
        long = True

    if dir_ == CXF_DEFAULT_LOG_DIR and not path.exists(CXF_DEFAULT_LOG_DIR):
        print('The default log directory `{}` does not exist.\n'
              'Consider specifying the directory to be listed as an argument.'.format(CXF_DEFAULT_LOG_DIR))
        quit(1)

    if not path.exists(dir_):
        print('Specified dir `{}` does not exist'.format(dir_))
        quit(1)

    all_trainings = _ls_print_listing(dir_, recursive, all_, long)

    if long and len(all_trainings) > 1:
        if not recursive:
            print()
        _ls_print_summary(all_trainings)

    if verbose and len(all_trainings) == 1:
        if not recursive:
            print()
        _ls_print_verbose(all_trainings[0])


Instruction:
How would you implement a function in Python 3 that
extends the accumulated variables with the given batch data.

Response:
def after_batch(self, stream_name: str, batch_data: Batch):
        """
        Extend the accumulated variables with the given batch data.

        :param stream_name: stream name; e.g. ``train`` or any other...
        :param batch_data: batch data = stream sources + model outputs
        :raise KeyError: if the variables to be aggregated are missing
        :raise TypeError: if the variable value is not iterable (e.g. it is only a scalar)
        """
        for variable in self._variables:
            if variable in batch_data:
                value = batch_data[variable]
                if not hasattr(value, '__iter__'):
                    raise TypeError('Variable `{}` to be accumulated is not iterable.'.format(variable))
                self._accumulator[stream_name][variable] += list(value)
            else:
                raise KeyError('Variable `{}` to be accumulated was not found in the batch data. '
                               'Available variables are `{}`.'.format(variable, batch_data.keys()))


Instruction:
Can you generate the documentation for the following Python 3 function
def invoke_dataset_method(config_path: str, method_name: str, output_root: str, cl_arguments: Iterable[str]) -> None:

    config = dataset = method = output_dir = None

    try:
        config_path = find_config(config_path)
        config = load_config(config_file=config_path, additional_args=cl_arguments)
        assert 'dataset' in config, '`dataset` section not present in the config'
        logging.debug('\tLoaded config: %s', config)
    except Exception as ex:  # pylint: disable=broad-except
        fallback('Loading config failed', ex)

    try:
        dataset = create_dataset(config)
    except Exception as ex:  # pylint: disable=broad-except
        fallback('Creating dataset failed', ex)

    try:
        method = getattr(dataset, method_name)
    except AttributeError as ex:
        fallback('Method `{}` not found in the dataset'.format(method_name), ex)

    try:
        method()
    except Exception as ex:  # pylint: disable=broad-except
        fallback('Exception occurred during method `{}` invocation'.format(method_name), ex)

Response: Create the specified dataset and invoke the specified method.


Instruction:
Make a summary of the following Python 3 code
def _signal_handler(self, *_) -> None:
        if self._num_signals == 0:
            logging.warning('Interrupt signal caught - training will be terminated')
            logging.warning('Another interrupt signal will terminate the program immediately')
            self._num_signals += 1
        else:
            logging.error('Another interrupt signal caught - terminating program immediately')
            sys.exit(2)

Response: Handles the interrupt signal.


Instruction:
How would you explain what the following Python 3 function does
def create_output_dir(config: dict, output_root: str, default_model_name: str='Unnamed') -> str:
    logging.info('Creating output dir')

    # create output dir
    model_name = default_model_name
    if 'name' not in config['model']:
        logging.warning('\tmodel.name not found in config, defaulting to: %s', model_name)
    else:
        model_name = config['model']['name']

    if not os.path.exists(output_root):
        logging.info('\tOutput root folder "%s" does not exist and will be created', output_root)
        os.makedirs(output_root)

    # keep trying to create new output dir until it succeeds
    # this is neccessary due to improbable yet possible output dir name conflicts
    while True:
        try:
            output_dir = path.join(output_root, '{}_{}_{}'.format(datetime.now().strftime('%Y-%m-%d-%H-%M-%S'),
                                                                  model_name, get_random_name()))
            os.mkdir(output_dir)
            break
        except OSError as ex:
            if ex.errno != errno.EEXIST:
                raise ex
    logging.info('\tOutput dir: %s', output_dir)

    # create file logger
    file_handler = logging.FileHandler(path.join(output_dir, CXF_LOG_FILE))
    file_handler.setFormatter(logging.Formatter(CXF_LOG_FORMAT, datefmt=CXF_LOG_DATE_FORMAT))
    logging.getLogger().addHandler(file_handler)

    return output_dir

Response: Create output_dir under the given output_root and create file logger logging to a file under this output_dir


Instruction:
Create a Python 3 function for
creating a dataset object according to the given config.

Response:
def create_dataset(config: dict, output_dir: Optional[str]=None) -> AbstractDataset:
    """
    Create a dataset object according to the given config.

    Dataset config section and the `output_dir` are passed to the constructor in a single YAML-encoded string.

    :param config: config dict with dataset config
    :param output_dir: path to the training output dir or None
    :return: dataset object
    """
    logging.info('Creating dataset')

    dataset_config = make_simple(config)['dataset']
    assert 'class' in dataset_config, '`dataset.class` not present in the config'
    dataset_module, dataset_class = parse_fully_qualified_name(dataset_config['class'])

    if 'output_dir' in dataset_config:
        raise ValueError('The `output_dir` key is reserved and can not be used in dataset configuration.')

    dataset_config = {'output_dir': output_dir, **config['dataset']}
    del dataset_config['class']

    dataset = create_object(dataset_module, dataset_class, args=(yaml_to_str(dataset_config),))
    logging.info('\t%s created', type(dataset).__name__)
    return dataset


Instruction:
Can you write a function in Python 3 where it
creates a model object from the checkpoint in output_dir.

Response:
def create_model(config: dict, output_dir: Optional[str], dataset: AbstractDataset,
                 restore_from: Optional[str]=None) -> AbstractModel:
    """
    Create a model object either from scratch of from the checkpoint in ``resume_dir``.

    Cxflow allows the following scenarios

    1. Create model: leave ``restore_from=None`` and specify ``class``;
    2. Restore model: specify ``restore_from`` which is a backend-specific path to (a directory with) the saved model.

    :param config: config dict with model config
    :param output_dir: path to the training output dir
    :param dataset: dataset object implementing the :py:class:`cxflow.datasets.AbstractDataset` concept
    :param restore_from: from whence the model should be restored (backend-specific information)
    :return: model object
    """

    logging.info('Creating a model')

    model_config = config['model']

    # workaround for ruamel.yaml expansion bug; see #222
    model_config = dict(model_config.items())

    assert 'class' in model_config, '`model.class` not present in the config'
    model_module, model_class = parse_fully_qualified_name(model_config['class'])

    # create model kwargs (without `class` and `name`)
    model_kwargs = {'dataset': dataset, 'log_dir': output_dir, 'restore_from': restore_from, **model_config}
    del model_kwargs['class']
    if 'name' in model_kwargs:
        del model_kwargs['name']

    try:
        model = create_object(model_module, model_class, kwargs=model_kwargs)
    except (ImportError, AttributeError) as ex:
        if restore_from is None:  # training case
            raise ImportError('Cannot create model from the specified model module `{}` and class `{}`.'.format(
                model_module, model_class)) from ex

        else:  # restore cases (resume, predict)
            logging.warning('Cannot create model from the specified model class `%s`.', model_config['class'])
            assert 'restore_fallback' in model_config, '`model.restore_fallback` not present in the config'
            logging.info('Trying to restore with fallback `%s` instead.', model_config['restore_fallback'])

            try:  # try fallback class
                fallback_module, fallback_class = parse_fully_qualified_name(model_config['restore_fallback'])
                model = create_object(fallback_module, fallback_class, kwargs=model_kwargs)
            except (ImportError, AttributeError) as ex:  # if fallback module/class specified but it fails
                raise ImportError('Cannot create model from the specified restore_fallback `{}`.'.format(
                    model_config['restore_fallback'],)) from ex

    logging.info('\t%s created', type(model).__name__)
    return model


Instruction:
How would you implement a function in Python 3 that
creates hooks in the order they appear in the config.

Response:
def create_hooks(config: dict, model: AbstractModel,
                 dataset: AbstractDataset, output_dir: str) -> Iterable[AbstractHook]:
    """
    Create hooks specified in ``config['hooks']`` list.

    Hook config entries may be one of the following types:

    .. code-block:: yaml
        :caption: A hook with default args specified only by its name as a string; e.g.

        hooks:
          - LogVariables
          - cxflow_tensorflow.WriteTensorBoard

    .. code-block:: yaml
        :caption: A hook with custom args as a dict name -> args; e.g.

        hooks:
          - StopAfter:
              n_epochs: 10

    :param config: config dict
    :param model: model object to be passed to the hooks
    :param dataset: dataset object to be passed to hooks
    :param output_dir: training output dir available to the hooks
    :return: list of hook objects
    """
    logging.info('Creating hooks')
    hooks = []
    if 'hooks' in config:
        for hook_config in config['hooks']:
            if isinstance(hook_config, str):
                hook_config = {hook_config: {}}
            assert len(hook_config) == 1, 'Hook configuration must have exactly one key (fully qualified name).'

            hook_path, hook_params = next(iter(hook_config.items()))
            if hook_params is None:
                logging.warning('\t\t Empty config of `%s` hook', hook_path)
                hook_params = {}

            # workaround for ruamel.yaml expansion bug; see #222
            hook_params = dict(hook_params.items())

            hook_module, hook_class = parse_fully_qualified_name(hook_path)
            # find the hook module if not specified
            if hook_module is None:
                hook_module = get_class_module(CXF_HOOKS_MODULE, hook_class)
                logging.debug('\tFound hook module `%s` for class `%s`', hook_module, hook_class)
                if hook_module is None:
                    raise ValueError('Can`t find hook module for hook class `{}`. '
                                     'Make sure it is defined under `{}` sub-modules.'
                                     .format(hook_class, CXF_HOOKS_MODULE))
            # create hook kwargs
            hook_kwargs = {'dataset': dataset, 'model': model, 'output_dir': output_dir, **hook_params}

            # create new hook
            try:
                hook = create_object(hook_module, hook_class, kwargs=hook_kwargs)
                hooks.append(hook)
                logging.info('\t%s created', type(hooks[-1]).__name__)
            except (ValueError, KeyError, TypeError, NameError, AttributeError, AssertionError, ImportError) as ex:
                logging.error('\tFailed to create a hook from config `%s`', hook_config)
                raise ex
    return hooks


Instruction:
Given the following Python 3 function, write the documentation
def run(config: dict, output_root: str, restore_from: str=None, eval: Optional[str]=None) -> None:

    output_dir = dataset = model = hooks = main_loop = None

    try:
        output_dir = create_output_dir(config=config, output_root=output_root)
    except Exception as ex:  # pylint: disable=broad-except
        fallback('Failed to create output dir', ex)

    try:
        dataset = create_dataset(config=config, output_dir=output_dir)
    except Exception as ex:  # pylint: disable=broad-except
        fallback('Creating dataset failed', ex)

    try:
        model = create_model(config=config, output_dir=output_dir, dataset=dataset, restore_from=restore_from)
    except Exception as ex:  # pylint: disable=broad-except
        fallback('Creating model failed', ex)

    try:  # save the config to file
        # modify the config so that it contains fallback information
        config['model']['restore_fallback'] = model.restore_fallback
        yaml_to_file(data=config, output_dir=output_dir, name=CXF_CONFIG_FILE)
    except Exception as ex:  # pylint: disable=broad-except
        fallback('Saving config failed', ex)

    try:
        hooks = create_hooks(config=config, model=model, dataset=dataset, output_dir=output_dir)
    except Exception as ex:  # pylint: disable=broad-except
        fallback('Creating hooks failed', ex)

    try:
        logging.info('Creating main loop')
        kwargs = config['main_loop'] if 'main_loop' in config else {}
        if eval is not None:
            kwargs['extra_streams'] = []
        main_loop = MainLoop(model=model, dataset=dataset, hooks=hooks, **kwargs)
    except Exception as ex:  # pylint: disable=broad-except
        fallback('Creating main loop failed', ex)

    if eval is not None:
        try:
            with main_loop:
                logging.info('Running the evaluation of stream `%s`', eval)
                main_loop.run_evaluation(eval)
        except Exception as ex:  # pylint: disable=broad-except
            fallback('Running the evaluation failed', ex)
    else:
        trace = TrainingTrace(output_dir)
        try:
            with main_loop:
                logging.info('Running the training')
                trace[TrainingTraceKeys.TRAIN_BEGIN] = datetime.now()
                main_loop.run_training(trace)
                trace[TrainingTraceKeys.EXIT_STATUS] = 0
        except Exception as ex:  # pylint: disable=broad-except
            trace[TrainingTraceKeys.EXIT_STATUS] = 1
            fallback('Running the training failed', ex)
        except SystemExit as ex:
            trace[TrainingTraceKeys.EXIT_STATUS] = ex.code
        finally:
            trace[TrainingTraceKeys.EPOCHS_DONE] = main_loop.epochs_done
            trace[TrainingTraceKeys.TRAIN_END] = datetime.now()

Response: Run the CXFlow training.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _safe_rmtree(dir_: str):
    try:
        rmtree(dir_)
    except OSError:
        logging.warning('\t\t Skipping %s due to OSError', dir_)
    else:
        logging.debug('\t\t Deleted %s', dir_)

Response: Wrap rmtree to inform user about ( un ) success.


Instruction:
Given the following Python 3 function, write the documentation
def _prune_subdirs(dir_: str) -> None:
    for logdir in [path.join(dir_, f) for f in listdir(dir_) if is_train_dir(path.join(dir_, f))]:
        for subdir in [path.join(logdir, f) for f in listdir(logdir) if path.isdir(path.join(logdir, f))]:
            _safe_rmtree(subdir)

Response: Delete all subdirs in training log dirs.


Instruction:
Can you create a Python 3 function that
deletes all training dirs with incomplete training artifacts or less than specified epochs done.

Response:
def _prune(dir_: str, epochs: int) -> None:
    """
    Delete all training dirs with incomplete training artifacts or with less than specified epochs done.

    :param dir_: dir with training log dirs
    :param epochs: minimum number of finished epochs to keep the training logs
    :return: number of log dirs pruned
    """
    for logdir in [path.join(dir_, f) for f in listdir(dir_) if path.isdir(path.join(dir_, f))]:
        if not is_train_dir(logdir):
            _safe_rmtree(logdir)
        else:
            trace_path = path.join(logdir, CXF_TRACE_FILE)
            try:
                epochs_done = TrainingTrace.from_file(trace_path)[TrainingTraceKeys.EPOCHS_DONE]
            except (KeyError, TypeError):
                epochs_done = 0
            if not epochs_done or epochs_done < epochs:
                _safe_rmtree(logdir)


Instruction:
Implement a Python 3 function for
pruning training log dirs contained in the given dir.

Response:
def prune_train_dirs(dir_: str, epochs: int, subdirs: bool) -> None:
    """
    Prune training log dirs contained in the given dir. The function is accessible through cxflow CLI `cxflow prune`.

    :param dir_: dir to be pruned
    :param epochs: minimum number of finished epochs to keep the training logs
    :param subdirs: delete subdirs in training log dirs
    """

    if dir_ == CXF_DEFAULT_LOG_DIR and not path.exists(CXF_DEFAULT_LOG_DIR):
        print('The default log directory `{}` does not exist.\n'
              'Consider specifying the directory to be listed as an argument.'.format(CXF_DEFAULT_LOG_DIR))
        quit(1)

    if not path.exists(dir_):
        print('Specified dir `{}` does not exist'.format(dir_))
        quit(1)

    _prune(dir_, epochs)
    if subdirs:
        _prune_subdirs(dir_)


Instruction:
Write a Python 3 function that can
list of model output names.

Response:
def output_names(self) -> Iterable[str]:
        """List of model output names."""
        self._load_models()
        return chain.from_iterable(map(lambda m: m.output_names, self._models))


Instruction:
Can you generate a brief explanation for the following Python 3 code
def run(self, batch: Batch, train: bool=False, stream: StreamWrapper=None) -> Batch:
        if train:
            raise ValueError('Ensemble model cannot be trained.')
        self._load_models()

        # run all the models in-order
        current_batch = dict(copy.deepcopy(batch))
        for model in self._models:
            current_batch.update(model.run(current_batch, False, None))

        return {key: current_batch[key] for key in self.output_names}

Response: Runs all the models in - order and return accumulated outputs.


Instruction:
Write a Python 3 script for
logging variables from the epoch data.

Response:
def _log_variables(self, epoch_data: EpochData):
        """
        Log variables from the epoch data.

        .. warning::
            At the moment, only scalars and dicts of scalars are properly formatted and logged.
            Other value types are ignored by default.

        One may set ``on_unknown_type`` to ``str`` in order to log all the variables anyways.

        :param epoch_data: epoch data to be logged
        :raise KeyError: if the specified variable is not found in the stream
        :raise TypeError: if the variable value is of unsupported type and ``self._on_unknown_type`` is set to ``error``
        """
        for stream_name in epoch_data.keys():
            stream_data = epoch_data[stream_name]
            variables = self._variables if self._variables is not None else stream_data.keys()
            for variable in variables:
                if variable not in stream_data:
                    raise KeyError('Variable `{}` to be logged was not found in the batch data for stream `{}`. '
                                   'Available variables are `{}`.'.format(variable, stream_name, stream_data.keys()))
                value = stream_data[variable]
                if np.isscalar(value):
                    logging.info('\t%s %s: %f', stream_name, variable, value)
                elif isinstance(value, dict):
                    keys = list(value.keys())
                    if len(keys) == 1:
                        logging.info('\t%s %s %s: %f', stream_name, variable, keys[0], value[keys[0]])
                    else:
                        logging.info('\t%s %s:', stream_name, variable)
                        for key, val in value.items():
                            logging.info('\t\t%s: %f', key, val)
                else:
                    if self._on_unknown_type == 'error':
                        raise TypeError('Variable type `{}` can not be logged. Variable name: `{}`.'
                                        .format(type(value).__name__, variable))
                    elif self._on_unknown_type == 'warn':
                        logging.warning('Variable type `%s` can not be logged. Variable name: `%s`.',
                                        type(value).__name__, variable)
                    elif self._on_unknown_type == 'str':
                        logging.info('\t%s %s: %s', stream_name, variable, value)


Instruction:
Given the following Python 3 function, write the documentation
def get_reference_end_from_cigar(reference_start, cigar):
        '''
        This returns the coordinate just past the last aligned base.
        This matches the behavior of pysam's reference_end method
        '''
        reference_end = reference_start

        # iterate through cigartuple
        for i in xrange(len(cigar)):
            k, n = cigar[i]
            if k in (0,2,3,7,8): # M, D, N, =, X
                reference_end += n
        return reference_end

Response: This returns the coordinate just past the last aligned base.