Instruction:
Can you generate a brief explanation for the following Python 3 code
def describe_training_job(self, TrainingJobName):
        if TrainingJobName not in LocalSagemakerClient._training_jobs:
            error_response = {'Error': {'Code': 'ValidationException', 'Message': 'Could not find local training job'}}
            raise ClientError(error_response, 'describe_training_job')
        else:
            return LocalSagemakerClient._training_jobs[TrainingJobName].describe()

Response: Describe a local training job.


Instruction:
Implement a function in Python 3 to
create a Local Model Object

Response:
def create_model(self, ModelName, PrimaryContainer, *args, **kwargs):  # pylint: disable=unused-argument
        """Create a Local Model Object

        Args:
            ModelName (str): the Model Name
            PrimaryContainer (dict): a SageMaker primary container definition
        """
        LocalSagemakerClient._models[ModelName] = _LocalModel(ModelName, PrimaryContainer)


Instruction:
Here you have a function in Python 3, explain what it does
def _initialize(self, boto_session, sagemaker_client, sagemaker_runtime_client):

        self.boto_session = boto_session or boto3.Session()
        self._region_name = self.boto_session.region_name

        if self._region_name is None:
            raise ValueError('Must setup local AWS configuration with a region supported by SageMaker.')

        self.sagemaker_client = LocalSagemakerClient(self)
        self.sagemaker_runtime_client = LocalSagemakerRuntimeClient(self.config)
        self.local_mode = True

Response: Initialize this Local SageMaker Session.


Instruction:
Write a Python 3 script for
preparing a container definition for the model.

Response:
def prepare_container_def(self, instance_type, accelerator_type=None):
        """Return a container definition with framework configuration set in model environment variables.

        Args:
            instance_type (str): The EC2 instance type to deploy this Model to. For example, 'ml.p2.xlarge'.
            accelerator_type (str): The Elastic Inference accelerator type to deploy to the instance for loading and
                making inferences to the model. For example, 'ml.eia1.medium'. Note: accelerator types are not
                supported by SKLearnModel.

        Returns:
            dict[str, str]: A container definition object usable with the CreateModel API.
        """
        if accelerator_type:
            raise ValueError("Accelerator types are not supported for Scikit-Learn.")

        deploy_image = self.image
        if not deploy_image:
            image_tag = "{}-{}-{}".format(self.framework_version, "cpu", self.py_version)
            deploy_image = default_framework_uri(
                self.__framework_name__,
                self.sagemaker_session.boto_region_name,
                image_tag)

        deploy_key_prefix = model_code_key_prefix(self.key_prefix, self.name, deploy_image)
        self._upload_code(deploy_key_prefix)
        deploy_env = dict(self.env)
        deploy_env.update(self._framework_env_vars())

        if self.model_server_workers:
            deploy_env[MODEL_SERVER_WORKERS_PARAM_NAME.upper()] = str(self.model_server_workers)
        return sagemaker.container_def(deploy_image, self.model_data, deploy_env)


Instruction:
Write a Python 3 script to
upload the training array and labels arrays to num_shards s3 objects stored in s3://bucket / key_prefix /.

Response:
def upload_numpy_to_s3_shards(num_shards, s3, bucket, key_prefix, array, labels=None):
    """Upload the training ``array`` and ``labels`` arrays to ``num_shards`` s3 objects,
    stored in "s3://``bucket``/``key_prefix``/"."""
    shards = _build_shards(num_shards, array)
    if labels is not None:
        label_shards = _build_shards(num_shards, labels)
    uploaded_files = []
    if key_prefix[-1] != '/':
        key_prefix = key_prefix + '/'
    try:
        for shard_index, shard in enumerate(shards):
            with tempfile.TemporaryFile() as file:
                if labels is not None:
                    write_numpy_to_dense_tensor(file, shard, label_shards[shard_index])
                else:
                    write_numpy_to_dense_tensor(file, shard)
                file.seek(0)
                shard_index_string = str(shard_index).zfill(len(str(len(shards))))
                file_name = "matrix_{}.pbr".format(shard_index_string)
                key = key_prefix + file_name
                logger.debug("Creating object {} in bucket {}".format(key, bucket))
                s3.Object(bucket, key).put(Body=file)
                uploaded_files.append(file_name)
        manifest_key = key_prefix + ".amazon.manifest"
        manifest_str = json.dumps(
            [{'prefix': 's3://{}/{}'.format(bucket, key_prefix)}] + uploaded_files)
        s3.Object(bucket, manifest_key).put(Body=manifest_str.encode('utf-8'))
        return "s3://{}/{}".format(bucket, manifest_key)
    except Exception as ex:  # pylint: disable=broad-except
        try:
            for file in uploaded_files:
                s3.Object(bucket, key_prefix + file).delete()
        finally:
            raise ex


Instruction:
Can you write a function in Python 3 where it
returns the docker registry for the given AWS region.

Response:
def registry(region_name, algorithm=None):
    """Return docker registry for the given AWS region

    Note: Not all the algorithms listed below have an Amazon Estimator implemented. For full list of
    pre-implemented Estimators, look at:

    https://github.com/aws/sagemaker-python-sdk/tree/master/src/sagemaker/amazon
    """
    if algorithm in [None, 'pca', 'kmeans', 'linear-learner', 'factorization-machines', 'ntm',
                     'randomcutforest', 'knn', 'object2vec', 'ipinsights']:
        account_id = {
            'us-east-1': '382416733822',
            'us-east-2': '404615174143',
            'us-west-2': '174872318107',
            'eu-west-1': '438346466558',
            'eu-central-1': '664544806723',
            'ap-northeast-1': '351501993468',
            'ap-northeast-2': '835164637446',
            'ap-southeast-2': '712309505854',
            'us-gov-west-1': '226302683700',
            'ap-southeast-1': '475088953585',
            'ap-south-1': '991648021394',
            'ca-central-1': '469771592824',
            'eu-west-2': '644912444149',
            'us-west-1': '632365934929',
            'us-iso-east-1': '490574956308',
        }[region_name]
    elif algorithm in ['lda']:
        account_id = {
            'us-east-1': '766337827248',
            'us-east-2': '999911452149',
            'us-west-2': '266724342769',
            'eu-west-1': '999678624901',
            'eu-central-1': '353608530281',
            'ap-northeast-1': '258307448986',
            'ap-northeast-2': '293181348795',
            'ap-southeast-2': '297031611018',
            'us-gov-west-1': '226302683700',
            'ap-southeast-1': '475088953585',
            'ap-south-1': '991648021394',
            'ca-central-1': '469771592824',
            'eu-west-2': '644912444149',
            'us-west-1': '632365934929',
            'us-iso-east-1': '490574956308',
        }[region_name]
    elif algorithm in ['forecasting-deepar']:
        account_id = {
            'us-east-1': '522234722520',
            'us-east-2': '566113047672',
            'us-west-2': '156387875391',
            'eu-west-1': '224300973850',
            'eu-central-1': '495149712605',
            'ap-northeast-1': '633353088612',
            'ap-northeast-2': '204372634319',
            'ap-southeast-2': '514117268639',
            'us-gov-west-1': '226302683700',
            'ap-southeast-1': '475088953585',
            'ap-south-1': '991648021394',
            'ca-central-1': '469771592824',
            'eu-west-2': '644912444149',
            'us-west-1': '632365934929',
            'us-iso-east-1': '490574956308',
        }[region_name]
    elif algorithm in ['xgboost', 'seq2seq', 'image-classification', 'blazingtext',
                       'object-detection', 'semantic-segmentation']:
        account_id = {
            'us-east-1': '811284229777',
            'us-east-2': '825641698319',
            'us-west-2': '433757028032',
            'eu-west-1': '685385470294',
            'eu-central-1': '813361260812',
            'ap-northeast-1': '501404015308',
            'ap-northeast-2': '306986355934',
            'ap-southeast-2': '544295431143',
            'us-gov-west-1': '226302683700',
            'ap-southeast-1': '475088953585',
            'ap-south-1': '991648021394',
            'ca-central-1': '469771592824',
            'eu-west-2': '644912444149',
            'us-west-1': '632365934929',
            'us-iso-east-1': '490574956308',
        }[region_name]
    elif algorithm in ['image-classification-neo', 'xgboost-neo']:
        account_id = {
            'us-west-2': '301217895009',
            'us-east-1': '785573368785',
            'eu-west-1': '802834080501',
            'us-east-2': '007439368137',
        }[region_name]
    else:
        raise ValueError('Algorithm class:{} does not have mapping to account_id with images'.format(algorithm))

    return get_ecr_image_uri_prefix(account_id, region_name)


Instruction:
Here you have a function in Python 3, explain what it does
def get_image_uri(region_name, repo_name, repo_version=1):
    repo = '{}:{}'.format(repo_name, repo_version)
    return '{}/{}'.format(registry(region_name, repo_name), repo)

Response: Return algorithm image URI for the given AWS region repository name and repository version


Instruction:
How would you code a function in Python 3 to
convert the job description to init params that can be handled by the class constructor.

Response:
def _prepare_init_params_from_job_description(cls, job_details, model_channel_name=None):
        """Convert the job description to init params that can be handled by the class constructor

        Args:
            job_details: the returned job details from a describe_training_job API call.
            model_channel_name (str): Name of the channel where pre-trained model data will be downloaded.

        Returns:
             dictionary: The transformed init_params

        """
        init_params = super(AmazonAlgorithmEstimatorBase, cls)._prepare_init_params_from_job_description(
            job_details, model_channel_name)

        # The hyperparam names may not be the same as the class attribute that holds them,
        # for instance: local_lloyd_init_method is called local_init_method. We need to map these
        # and pass the correct name to the constructor.
        for attribute, value in cls.__dict__.items():
            if isinstance(value, hp):
                if value.name in init_params['hyperparameters']:
                    init_params[attribute] = init_params['hyperparameters'][value.name]

        del init_params['hyperparameters']
        del init_params['image']
        return init_params


Instruction:
Write a Python 3 script to
set hyperparameters needed for training.

Response:
def _prepare_for_training(self, records, mini_batch_size=None, job_name=None):
        """Set hyperparameters needed for training.

        Args:
            * records (:class:`~RecordSet`): The records to train this ``Estimator`` on.
            * mini_batch_size (int or None): The size of each mini-batch to use when training. If ``None``, a
                default value will be used.
            * job_name (str): Name of the training job to be created. If not specified, one is generated,
                using the base name given to the constructor if applicable.
        """
        super(AmazonAlgorithmEstimatorBase, self)._prepare_for_training(job_name=job_name)

        feature_dim = None

        if isinstance(records, list):
            for record in records:
                if record.channel == 'train':
                    feature_dim = record.feature_dim
                    break
            if feature_dim is None:
                raise ValueError('Must provide train channel.')
        else:
            feature_dim = records.feature_dim

        self.feature_dim = feature_dim
        self.mini_batch_size = mini_batch_size


Instruction:
Write a Python 3 function for
fitting this Estimator on serialized Record objects stored in S3 and store them in S3.

Response:
def fit(self, records, mini_batch_size=None, wait=True, logs=True, job_name=None):
        """Fit this Estimator on serialized Record objects, stored in S3.

        ``records`` should be an instance of :class:`~RecordSet`. This defines a collection of
        S3 data files to train this ``Estimator`` on.

        Training data is expected to be encoded as dense or sparse vectors in the "values" feature
        on each Record. If the data is labeled, the label is expected to be encoded as a list of
        scalas in the "values" feature of the Record label.

        More information on the Amazon Record format is available at:
        https://docs.aws.amazon.com/sagemaker/latest/dg/cdf-training.html

        See :meth:`~AmazonAlgorithmEstimatorBase.record_set` to construct a ``RecordSet`` object
        from :class:`~numpy.ndarray` arrays.

        Args:
            records (:class:`~RecordSet`): The records to train this ``Estimator`` on
            mini_batch_size (int or None): The size of each mini-batch to use when training. If ``None``, a
                default value will be used.
            wait (bool): Whether the call should wait until the job completes (default: True).
            logs (bool): Whether to show the logs produced by the job.
                Only meaningful when wait is True (default: True).
            job_name (str): Training job name. If not specified, the estimator generates a default job name,
                based on the training image name and current timestamp.
        """
        self._prepare_for_training(records, job_name=job_name, mini_batch_size=mini_batch_size)

        self.latest_training_job = _TrainingJob.start_new(self, records)
        if wait:
            self.latest_training_job.wait(logs=logs)


Instruction:
Can you write a function in Python 3 where it
builds a RecordSet from a 2D numpy matrix and label vector.

Response:
def record_set(self, train, labels=None, channel="train"):
        """Build a :class:`~RecordSet` from a numpy :class:`~ndarray` matrix and label vector.

        For the 2D ``ndarray`` ``train``, each row is converted to a :class:`~Record` object.
        The vector is stored in the "values" entry of the ``features`` property of each Record.
        If ``labels`` is not None, each corresponding label is assigned to the "values" entry
        of the ``labels`` property of each Record.

        The collection of ``Record`` objects are protobuf serialized and uploaded to new
        S3 locations. A manifest file is generated containing the list of objects created and
        also stored in S3.

        The number of S3 objects created is controlled by the ``train_instance_count`` property
        on this Estimator. One S3 object is created per training instance.

        Args:
            train (numpy.ndarray): A 2D numpy array of training data.
            labels (numpy.ndarray): A 1D numpy array of labels. Its length must be equal to the
               number of rows in ``train``.
            channel (str): The SageMaker TrainingJob channel this RecordSet should be assigned to.
        Returns:
            RecordSet: A RecordSet referencing the encoded, uploading training and label data.
        """
        s3 = self.sagemaker_session.boto_session.resource('s3')
        parsed_s3_url = urlparse(self.data_location)
        bucket, key_prefix = parsed_s3_url.netloc, parsed_s3_url.path
        key_prefix = key_prefix + '{}-{}/'.format(type(self).__name__, sagemaker_timestamp())
        key_prefix = key_prefix.lstrip('/')
        logger.debug('Uploading to bucket {} and key_prefix {}'.format(bucket, key_prefix))
        manifest_s3_file = upload_numpy_to_s3_shards(self.train_instance_count, s3, bucket, key_prefix, train, labels)
        logger.debug("Created manifest file {}".format(manifest_s3_file))
        return RecordSet(manifest_s3_file, num_records=train.shape[0], feature_dim=train.shape[1], channel=channel)


Instruction:
Can you tell what is the following Python 3 function doing
def get_data_source_instance(data_source, sagemaker_session):
    parsed_uri = urlparse(data_source)
    if parsed_uri.scheme == 'file':
        return LocalFileDataSource(parsed_uri.netloc + parsed_uri.path)
    elif parsed_uri.scheme == 's3':
        return S3DataSource(parsed_uri.netloc, parsed_uri.path, sagemaker_session)

Response: Return an instance of a Data Source that can handle the provided data_source URI.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def get_splitter_instance(split_type):
    if split_type is None:
        return NoneSplitter()
    elif split_type == 'Line':
        return LineSplitter()
    elif split_type == 'RecordIO':
        return RecordIOSplitter()
    else:
        raise ValueError('Invalid Split Type: %s' % split_type)

Response:
Returns an Instance of a Splitter according to
            the specified split_type.


Instruction:
Can you write a function in Python 3 where it
returns an Instance of a BatchStrategy according to strategy

Response:
def get_batch_strategy_instance(strategy, splitter):
    """Return an Instance of :class:`sagemaker.local.data.BatchStrategy` according to `strategy`

    Args:
        strategy (str): Either 'SingleRecord' or 'MultiRecord'
        splitter (:class:`sagemaker.local.data.Splitter): splitter to get the data from.

    Returns
        :class:`sagemaker.local.data.BatchStrategy`: an Instance of a BatchStrategy
    """
    if strategy == 'SingleRecord':
        return SingleRecordStrategy(splitter)
    elif strategy == 'MultiRecord':
        return MultiRecordStrategy(splitter)
    else:
        raise ValueError('Invalid Batch Strategy: %s - Valid Strategies: "SingleRecord", "MultiRecord"')


Instruction:
Implement a Python 3 function for
retrieving the list of absolute paths to all the files in this data source.

Response:
def get_file_list(self):
        """Retrieve the list of absolute paths to all the files in this data source.

        Returns:
             List[str] List of absolute paths.
        """
        if os.path.isdir(self.root_path):
            return [os.path.join(self.root_path, f) for f in os.listdir(self.root_path)
                    if os.path.isfile(os.path.join(self.root_path, f))]
        else:
            return [self.root_path]


Instruction:
How would you explain what the following Python 3 function does
def get_root_dir(self):
        if os.path.isdir(self.root_path):
            return self.root_path
        else:
            return os.path.dirname(self.root_path)

Response: Retrieve the absolute path to the root directory of this data source.


Instruction:
Create a Python 3 function for
splitting a file into individual RecordIO records using a specific strategy

Response:
def split(self, file):
        """Split a file into records using a specific strategy

        This RecordIOSplitter splits the data into individual RecordIO records.

        Args:
            file (str): path to the file to split

        Returns: generator for the individual records that were split from the file
        """
        with open(file, 'rb') as f:
            for record in sagemaker.amazon.common.read_recordio(f):
                yield record


Instruction:
Can you tell what is the following Python 3 function doing
def pad(self, file, size=6):
        buffer = ''
        for element in self.splitter.split(file):
            if _payload_size_within_limit(buffer + element, size):
                buffer += element
            else:
                tmp = buffer
                buffer = element
                yield tmp
        if _validate_payload_size(buffer, size):
            yield buffer

Response: Returns a generator that yields the specified size of the records in file.


Instruction:
Can you generate the documentation for the following Python 3 function
def pad(self, file, size=6):
        for element in self.splitter.split(file):
            if _validate_payload_size(element, size):
                yield element

Response: Returns a generator that yields all the records in the specified size.


Instruction:
Write a Python 3 function that can
return hyperparameters used by your custom Chainer code during training.

Response:
def hyperparameters(self):
        """Return hyperparameters used by your custom Chainer code during training."""
        hyperparameters = super(Chainer, self).hyperparameters()

        additional_hyperparameters = {Chainer._use_mpi: self.use_mpi,
                                      Chainer._num_processes: self.num_processes,
                                      Chainer._process_slots_per_host: self.process_slots_per_host,
                                      Chainer._additional_mpi_options: self.additional_mpi_options}

        # remove unset keys.
        additional_hyperparameters = {k: v for k, v in additional_hyperparameters.items() if v}
        hyperparameters.update(Framework._json_encode_hyperparameters(additional_hyperparameters))
        return hyperparameters


Instruction:
Write a Python 3 script for
creating a SageMaker ChainerModel object that can be deployed to an Endpoint.

Response:
def create_model(self, model_server_workers=None, role=None, vpc_config_override=VPC_CONFIG_DEFAULT):
        """Create a SageMaker ``ChainerModel`` object that can be deployed to an ``Endpoint``.

        Args:
            role (str): The ``ExecutionRoleArn`` IAM Role ARN for the ``Model``, which is also used during
                transform jobs. If not specified, the role from the Estimator will be used.
            model_server_workers (int): Optional. The number of worker processes used by the inference server.
                If None, server will use one worker per vCPU.
            vpc_config_override (dict[str, list[str]]): Optional override for VpcConfig set on the model.
                Default: use subnets and security groups from this Estimator.
                * 'Subnets' (list[str]): List of subnet ids.
                * 'SecurityGroupIds' (list[str]): List of security group ids.

        Returns:
            sagemaker.chainer.model.ChainerModel: A SageMaker ``ChainerModel`` object.
                See :func:`~sagemaker.chainer.model.ChainerModel` for full details.
        """
        role = role or self.role
        return ChainerModel(self.model_data, role, self.entry_point, source_dir=self._model_source_dir(),
                            enable_cloudwatch_metrics=self.enable_cloudwatch_metrics, name=self._current_job_name,
                            container_log_level=self.container_log_level, code_location=self.code_location,
                            py_version=self.py_version, framework_version=self.framework_version,
                            model_server_workers=model_server_workers, image=self.image_name,
                            sagemaker_session=self.sagemaker_session,
                            vpc_config=self.get_vpc_config(vpc_config_override), dependencies=self.dependencies)


Instruction:
Write a Python 3 script to
convert the job description to init params that can be handled by the class constructor.

Response:
def _prepare_init_params_from_job_description(cls, job_details, model_channel_name=None):
        """Convert the job description to init params that can be handled by the class constructor

        Args:
            job_details: the returned job details from a describe_training_job API call.
            model_channel_name (str): Name of the channel where pre-trained model data will be downloaded.

        Returns:
             dictionary: The transformed init_params

        """
        init_params = super(Chainer, cls)._prepare_init_params_from_job_description(job_details, model_channel_name)

        for argument in [Chainer._use_mpi, Chainer._num_processes, Chainer._process_slots_per_host,
                         Chainer._additional_mpi_options]:

            value = init_params['hyperparameters'].pop(argument, None)
            if value:
                init_params[argument[len('sagemaker_'):]] = value

        image_name = init_params.pop('image')
        framework, py_version, tag, _ = framework_name_from_image(image_name)

        if not framework:
            # If we were unable to parse the framework name from the image it is not one of our
            # officially supported images, in this case just add the image to the init params.
            init_params['image_name'] = image_name
            return init_params

        init_params['py_version'] = py_version
        init_params['framework_version'] = framework_version_from_tag(tag)

        training_job_name = init_params['base_job_name']

        if framework != cls.__framework_name__:
            raise ValueError("Training job: {} didn't use image for requested framework".format(training_job_name))
        return init_params


Instruction:
Can you tell what is the following Python 3 function doing
def write_numpy_to_dense_tensor(file, array, labels=None):

    # Validate shape of array and labels, resolve array and label types
    if not len(array.shape) == 2:
        raise ValueError("Array must be a Matrix")
    if labels is not None:
        if not len(labels.shape) == 1:
            raise ValueError("Labels must be a Vector")
        if labels.shape[0] not in array.shape:
            raise ValueError("Label shape {} not compatible with array shape {}".format(
                             labels.shape, array.shape))
        resolved_label_type = _resolve_type(labels.dtype)
    resolved_type = _resolve_type(array.dtype)

    # Write each vector in array into a Record in the file object
    record = Record()
    for index, vector in enumerate(array):
        record.Clear()
        _write_feature_tensor(resolved_type, record, vector)
        if labels is not None:
            _write_label_tensor(resolved_label_type, record, labels[index])
        _write_recordio(file, record.SerializeToString())

Response: Writes a numpy array to a dense tensor.


Instruction:
Can you implement a function in Python 3 that
writes a scipy sparse matrix to a sparse tensor.

Response:
def write_spmatrix_to_sparse_tensor(file, array, labels=None):
    """Writes a scipy sparse matrix to a sparse tensor"""

    if not issparse(array):
        raise TypeError("Array must be sparse")

    # Validate shape of array and labels, resolve array and label types
    if not len(array.shape) == 2:
        raise ValueError("Array must be a Matrix")
    if labels is not None:
        if not len(labels.shape) == 1:
            raise ValueError("Labels must be a Vector")
        if labels.shape[0] not in array.shape:
            raise ValueError("Label shape {} not compatible with array shape {}".format(
                             labels.shape, array.shape))
        resolved_label_type = _resolve_type(labels.dtype)
    resolved_type = _resolve_type(array.dtype)

    csr_array = array.tocsr()
    n_rows, n_cols = csr_array.shape

    record = Record()
    for row_idx in range(n_rows):
        record.Clear()
        row = csr_array.getrow(row_idx)
        # Write values
        _write_feature_tensor(resolved_type, record, row.data)
        # Write keys
        _write_keys_tensor(resolved_type, record, row.indices.astype(np.uint64))

        # Write labels
        if labels is not None:
            _write_label_tensor(resolved_label_type, record, labels[row_idx])

        # Write shape
        _write_shape(resolved_type, record, n_cols)

        _write_recordio(file, record.SerializeToString())


Instruction:
Can you create a Python 3 function that
writes a single data point as a RecordIO record to the given file.

Response:
def _write_recordio(f, data):
    """Writes a single data point as a RecordIO record to the given file."""
    length = len(data)
    f.write(struct.pack('I', _kmagic))
    f.write(struct.pack('I', length))
    pad = (((length + 3) >> 2) << 2) - length
    f.write(data)
    f.write(padding[pad])


Instruction:
Can you create a Python 3 function that
prepares a container definition for deploying this model to a specific instance type.

Response:
def prepare_container_def(self, instance_type, accelerator_type=None):  # pylint: disable=unused-argument
        """Return a dict created by ``sagemaker.container_def()`` for deploying this model to a specified instance type.

        Subclasses can override this to provide custom container definitions for
        deployment to a specific instance type. Called by ``deploy()``.

        Args:
            instance_type (str): The EC2 instance type to deploy this Model to. For example, 'ml.p2.xlarge'.
            accelerator_type (str): The Elastic Inference accelerator type to deploy to the instance for loading and
                making inferences to the model. For example, 'ml.eia1.medium'.

        Returns:
            dict: A container definition object usable with the CreateModel API.
        """
        return sagemaker.container_def(self.image, self.model_data, self.env)


Instruction:
Write a Python 3 script to
create a SageMaker Model Entity MimeType

Response:
def _create_sagemaker_model(self, instance_type, accelerator_type=None, tags=None):
        """Create a SageMaker Model Entity

        Args:
            instance_type (str): The EC2 instance type that this Model will be used for, this is only
                used to determine if the image needs GPU support or not.
            accelerator_type (str): Type of Elastic Inference accelerator to attach to an endpoint for model loading
                and inference, for example, 'ml.eia1.medium'. If not specified, no Elastic Inference accelerator
                will be attached to the endpoint.
            tags(List[dict[str, str]]): Optional. The list of tags to add to the model. Example:
                    >>> tags = [{'Key': 'tagname', 'Value': 'tagvalue'}]
                    For more information about tags, see https://boto3.amazonaws.com/v1/documentation\
                    /api/latest/reference/services/sagemaker.html#SageMaker.Client.add_tags

        """
        container_def = self.prepare_container_def(instance_type, accelerator_type=accelerator_type)
        self.name = self.name or utils.name_from_image(container_def['Image'])
        enable_network_isolation = self.enable_network_isolation()
        self.sagemaker_session.create_model(self.name, self.role,
                                            container_def, vpc_config=self.vpc_config,
                                            enable_network_isolation=enable_network_isolation,
                                            tags=tags)


Instruction:
Can you tell what is the following Python 3 function doing
def compile(self, target_instance_family, input_shape, output_path, role,
                tags=None, job_name=None, compile_max_run=5 * 60, framework=None, framework_version=None):
        """Compile this ``Model`` with SageMaker Neo.

        Args:
            target_instance_family (str): Identifies the device that you want to run your model after compilation, for
                example: ml_c5. Allowed strings are: ml_c5, ml_m5, ml_c4, ml_m4, jetsontx1, jetsontx2, ml_p2, ml_p3,
                deeplens, rasp3b
            input_shape (dict): Specifies the name and shape of the expected inputs for your trained model in json
                dictionary form, for example: {'data':[1,3,1024,1024]}, or {'var1': [1,1,28,28], 'var2':[1,1,28,28]}
            output_path (str): Specifies where to store the compiled model
            role (str): Execution role
            tags (list[dict]): List of tags for labeling a compilation job. For more, see
                https://docs.aws.amazon.com/sagemaker/latest/dg/API_Tag.html.
            job_name (str): The name of the compilation job
            compile_max_run (int): Timeout in seconds for compilation (default: 3 * 60).
                After this amount of time Amazon SageMaker Neo terminates the compilation job regardless of its
                current status.
            framework (str): The framework that is used to train the original model. Allowed values: 'mxnet',
                'tensorflow', 'pytorch', 'onnx', 'xgboost'
            framework_version (str)
        Returns:
            sagemaker.model.Model: A SageMaker ``Model`` object. See :func:`~sagemaker.model.Model` for full details.
        """
        framework = self._framework() or framework
        if framework is None:
            raise ValueError("You must specify framework, allowed values {}".format(NEO_ALLOWED_FRAMEWORKS))
        if framework not in NEO_ALLOWED_FRAMEWORKS:
            raise ValueError("You must provide valid framework, allowed values {}".format(NEO_ALLOWED_FRAMEWORKS))
        if job_name is None:
            raise ValueError("You must provide a compilation job name")

        framework = framework.upper()
        framework_version = self._get_framework_version() or framework_version

        config = self._compilation_job_config(target_instance_family, input_shape, output_path, role,
                                              compile_max_run, job_name, framework, tags)
        self.sagemaker_session.compile_model(**config)
        job_status = self.sagemaker_session.wait_for_compilation_job(job_name)
        self.model_data = job_status['ModelArtifacts']['S3ModelArtifacts']
        if target_instance_family.startswith('ml_'):
            self.image = self._neo_image(self.sagemaker_session.boto_region_name, target_instance_family, framework,
                                         framework_version)
            self._is_compiled_model = True
        else:
            LOGGER.warning("The intance type {} is not supported to deploy via SageMaker,"
                           "please deploy the model on the device by yourself.".format(target_instance_family))
        return self

Response: Compile this model with SageMaker Neo.


Instruction:
Write a Python 3 function that can
deploy this model to an Amazon SageMaker Endpoint and optionally return a Predictor.

Response:
def deploy(self, initial_instance_count, instance_type, accelerator_type=None, endpoint_name=None,
               update_endpoint=False, tags=None, kms_key=None):
        """Deploy this ``Model`` to an ``Endpoint`` and optionally return a ``Predictor``.

        Create a SageMaker ``Model`` and ``EndpointConfig``, and deploy an ``Endpoint`` from this ``Model``.
        If ``self.predictor_cls`` is not None, this method returns a the result of invoking
        ``self.predictor_cls`` on the created endpoint name.

        The name of the created model is accessible in the ``name`` field of this ``Model`` after deploy returns

        The name of the created endpoint is accessible in the ``endpoint_name``
        field of this ``Model`` after deploy returns.

        Args:
            instance_type (str): The EC2 instance type to deploy this Model to. For example, 'ml.p2.xlarge'.
            initial_instance_count (int): The initial number of instances to run in the
                ``Endpoint`` created from this ``Model``.
            accelerator_type (str): Type of Elastic Inference accelerator to deploy this model for model loading
                and inference, for example, 'ml.eia1.medium'. If not specified, no Elastic Inference accelerator
                will be attached to the endpoint.
                For more information: https://docs.aws.amazon.com/sagemaker/latest/dg/ei.html
            endpoint_name (str): The name of the endpoint to create (default: None).
                If not specified, a unique endpoint name will be created.
            update_endpoint (bool): Flag to update the model in an existing Amazon SageMaker endpoint.
                If True, this will deploy a new EndpointConfig to an already existing endpoint and delete resources
                corresponding to the previous EndpointConfig. If False, a new endpoint will be created. Default: False
            tags(List[dict[str, str]]): The list of tags to attach to this specific endpoint.
            kms_key (str): The ARN of the KMS key that is used to encrypt the data on the
                storage volume attached to the instance hosting the endpoint.

        Returns:
            callable[string, sagemaker.session.Session] or None: Invocation of ``self.predictor_cls`` on
                the created endpoint name, if ``self.predictor_cls`` is not None. Otherwise, return None.
        """
        if not self.sagemaker_session:
            if instance_type in ('local', 'local_gpu'):
                self.sagemaker_session = local.LocalSession()
            else:
                self.sagemaker_session = session.Session()

        if self.role is None:
            raise ValueError("Role can not be null for deploying a model")

        compiled_model_suffix = '-'.join(instance_type.split('.')[:-1])
        if self._is_compiled_model:
            self.name += compiled_model_suffix

        self._create_sagemaker_model(instance_type, accelerator_type, tags)
        production_variant = sagemaker.production_variant(self.name, instance_type, initial_instance_count,
                                                          accelerator_type=accelerator_type)
        if endpoint_name:
            self.endpoint_name = endpoint_name
        else:
            self.endpoint_name = self.name
            if self._is_compiled_model and not self.endpoint_name.endswith(compiled_model_suffix):
                self.endpoint_name += compiled_model_suffix

        if update_endpoint:
            endpoint_config_name = self.sagemaker_session.create_endpoint_config(
                name=self.name,
                model_name=self.name,
                initial_instance_count=initial_instance_count,
                instance_type=instance_type,
                accelerator_type=accelerator_type,
                tags=tags,
                kms_key=kms_key)
            self.sagemaker_session.update_endpoint(self.endpoint_name, endpoint_config_name)
        else:
            self.sagemaker_session.endpoint_from_production_variants(self.endpoint_name, [production_variant],
                                                                     tags, kms_key)

        if self.predictor_cls:
            return self.predictor_cls(self.endpoint_name, self.sagemaker_session)


Instruction:
How would you implement a function in Python 3 that
returns a Transformer that uses this Model.

Response:
def transformer(self, instance_count, instance_type, strategy=None, assemble_with=None, output_path=None,
                    output_kms_key=None, accept=None, env=None, max_concurrent_transforms=None,
                    max_payload=None, tags=None, volume_kms_key=None):
        """Return a ``Transformer`` that uses this Model.

        Args:
            instance_count (int): Number of EC2 instances to use.
            instance_type (str): Type of EC2 instance to use, for example, 'ml.c4.xlarge'.
            strategy (str): The strategy used to decide how to batch records in a single request (default: None).
                Valid values: 'MULTI_RECORD' and 'SINGLE_RECORD'.
            assemble_with (str): How the output is assembled (default: None). Valid values: 'Line' or 'None'.
            output_path (str): S3 location for saving the transform result. If not specified, results are stored to
                a default bucket.
            output_kms_key (str): Optional. KMS key ID for encrypting the transform output (default: None).
            accept (str): The content type accepted by the endpoint deployed during the transform job.
            env (dict): Environment variables to be set for use during the transform job (default: None).
            max_concurrent_transforms (int): The maximum number of HTTP requests to be made to
                each individual transform container at one time.
            max_payload (int): Maximum size of the payload in a single HTTP request to the container in MB.
            tags (list[dict]): List of tags for labeling a transform job. If none specified, then the tags used for
                the training job are used for the transform job.
            volume_kms_key (str): Optional. KMS key ID for encrypting the volume attached to the ML
                compute instance (default: None).
        """
        self._create_sagemaker_model(instance_type)
        if self.enable_network_isolation():
            env = None

        return Transformer(self.name, instance_count, instance_type, strategy=strategy, assemble_with=assemble_with,
                           output_path=output_path, output_kms_key=output_kms_key, accept=accept,
                           max_concurrent_transforms=max_concurrent_transforms, max_payload=max_payload,
                           env=env, tags=tags, base_transform_job_name=self.name,
                           volume_kms_key=volume_kms_key, sagemaker_session=self.sagemaker_session)


Instruction:
Implement a Python 3 function for
returning a SageMaker container definition object with framework configuration set in model environment variables.

Response:
def prepare_container_def(self, instance_type, accelerator_type=None):  # pylint disable=unused-argument
        """Return a container definition with framework configuration set in model environment variables.

        This also uploads user-supplied code to S3.

        Args:
            instance_type (str): The EC2 instance type to deploy this Model to. For example, 'ml.p2.xlarge'.
            accelerator_type (str): The Elastic Inference accelerator type to deploy to the instance for loading and
                making inferences to the model. For example, 'ml.eia1.medium'.

        Returns:
            dict[str, str]: A container definition object usable with the CreateModel API.
        """
        deploy_key_prefix = fw_utils.model_code_key_prefix(self.key_prefix, self.name, self.image)
        self._upload_code(deploy_key_prefix)
        deploy_env = dict(self.env)
        deploy_env.update(self._framework_env_vars())
        return sagemaker.container_def(self.image, self.model_data, deploy_env)


Instruction:
Write a Python 3 script for
creating a SageMaker Model Entity and return the ModelPackage.

Response:
def _create_sagemaker_model(self, *args):  # pylint: disable=unused-argument
        """Create a SageMaker Model Entity

        Args:
            *args: Arguments coming from the caller. This class
                does not require any so they are ignored.
        """
        if self.algorithm_arn:
            # When ModelPackage is created using an algorithm_arn we need to first
            # create a ModelPackage. If we had already created one then its fine to re-use it.
            if self._created_model_package_name is None:
                model_package_name = self._create_sagemaker_model_package()
                self.sagemaker_session.wait_for_model_package(model_package_name)
                self._created_model_package_name = model_package_name
            model_package_name = self._created_model_package_name
        else:
            # When a ModelPackageArn is provided we just create the Model
            model_package_name = self.model_package_arn

        container_def = {
            'ModelPackageName': model_package_name,
        }

        if self.env != {}:
            container_def['Environment'] = self.env

        model_package_short_name = model_package_name.split('/')[-1]
        enable_network_isolation = self.enable_network_isolation()
        self.name = self.name or utils.name_from_base(model_package_short_name)
        self.sagemaker_session.create_model(self.name, self.role, container_def,
                                            vpc_config=self.vpc_config,
                                            enable_network_isolation=enable_network_isolation)


Instruction:
How would you implement a function in Python 3 that
returns the inference from the specified endpoint.

Response:
def predict(self, data, initial_args=None):
        """Return the inference from the specified endpoint.

        Args:
            data (object): Input data for which you want the model to provide inference.
                If a serializer was specified when creating the RealTimePredictor, the result of the
                serializer is sent as input data. Otherwise the data must be sequence of bytes, and
                the predict method then sends the bytes in the request body as is.
            initial_args (dict[str,str]): Optional. Default arguments for boto3
                ``invoke_endpoint`` call. Default is None (no default arguments).

        Returns:
            object: Inference for the given input. If a deserializer was specified when creating
                the RealTimePredictor, the result of the deserializer is returned. Otherwise the response
                returns the sequence of bytes as is.
        """

        request_args = self._create_request_args(data, initial_args)
        response = self.sagemaker_session.sagemaker_runtime_client.invoke_endpoint(**request_args)
        return self._handle_response(response)


Instruction:
Here you have a function in Python 3, explain what it does
def delete_endpoint(self, delete_endpoint_config=True):
        if delete_endpoint_config:
            self._delete_endpoint_config()

        self.sagemaker_session.delete_endpoint(self.endpoint)

Response: Delete the Amazon SageMaker endpoint backing this predictor.


Instruction:
Make a summary of the following Python 3 code
def delete_model(self):
        request_failed = False
        failed_models = []
        for model_name in self._model_names:
            try:
                self.sagemaker_session.delete_model(model_name)
            except Exception:  # pylint: disable=broad-except
                request_failed = True
                failed_models.append(model_name)

        if request_failed:
            raise Exception('One or more models cannot be deleted, please retry. \n'
                            'Failed models: {}'.format(', '.join(failed_models)))

Response: Deletes the Amazon SageMaker models backing this predictor.


Instruction:
Implement a function in Python 3 to
clear the local cache of all local data structures.

Response:
def clear_cache(self):
        """Clear the object of all local caches of API methods.
        """
        super(HyperparameterTuningJobAnalytics, self).clear_cache()
        self._tuning_job_describe_result = None
        self._training_job_summaries = None


Instruction:
Can you generate the documentation for the following Python 3 function
def description(self, force_refresh=False):
        if force_refresh:
            self.clear_cache()
        if not self._tuning_job_describe_result:
            self._tuning_job_describe_result = self._sage_client.describe_hyper_parameter_tuning_job(
                HyperParameterTuningJobName=self.name
            )
        return self._tuning_job_describe_result

Response: Get the description of the hyperparameter tuning job.


Instruction:
Here you have a function in Python 3, explain what it does
def training_job_summaries(self, force_refresh=False):
        if force_refresh:
            self.clear_cache()
        if self._training_job_summaries is not None:
            return self._training_job_summaries
        output = []
        next_args = {}
        for count in range(100):
            logging.debug("Calling list_training_jobs_for_hyper_parameter_tuning_job %d" % count)
            raw_result = self._sage_client.list_training_jobs_for_hyper_parameter_tuning_job(
                HyperParameterTuningJobName=self.name, MaxResults=100, **next_args
            )
            new_output = raw_result['TrainingJobSummaries']
            output.extend(new_output)
            logging.debug("Got %d more TrainingJobs. Total so far: %d" % (len(new_output), len(output)))
            if ('NextToken' in raw_result) and (len(new_output) > 0):
                next_args['NextToken'] = raw_result['NextToken']
            else:
                break
        self._training_job_summaries = output
        return output

Response: A list of all training jobs for a hyperparameter tuning job.


Instruction:
Make a summary of the following Python 3 code
def clear_cache(self):
        super(TrainingJobAnalytics, self).clear_cache()
        self._data = defaultdict(list)
        self._time_interval = self._determine_timeinterval()

Response:
Clear the local cache of all API methods so
        is not used.


Instruction:
Explain what the following Python 3 code does
def _determine_timeinterval(self):
        description = self._sage_client.describe_training_job(TrainingJobName=self.name)
        start_time = self._start_time or description[u'TrainingStartTime']  # datetime object
        # Incrementing end time by 1 min since CloudWatch drops seconds before finding the logs.
        # This results in logs being searched in the time range in which the correct log line was not present.
        # Example - Log time - 2018-10-22 08:25:55
        #           Here calculated end time would also be 2018-10-22 08:25:55 (without 1 min addition)
        #           CW will consider end time as 2018-10-22 08:25 and will not be able to search the correct log.
        end_time = self._end_time or description.get(
            u'TrainingEndTime', datetime.datetime.utcnow()) + datetime.timedelta(minutes=1)

        return {
            'start_time': start_time,
            'end_time': end_time,
        }

Response: Return a dictionary with two datetime objects start_time and end_time covering the interval of the training job.


Instruction:
Write a Python 3 function for
fetching all the values of a named metric and add them to _data

Response:
def _fetch_metric(self, metric_name):
        """Fetch all the values of a named metric, and add them to _data
        """
        request = {
            'Namespace': self.CLOUDWATCH_NAMESPACE,
            'MetricName': metric_name,
            'Dimensions': [
                {
                    'Name': 'TrainingJobName',
                    'Value': self.name
                }
            ],
            'StartTime': self._time_interval['start_time'],
            'EndTime': self._time_interval['end_time'],
            'Period': self._period,
            'Statistics': ['Average'],
        }
        raw_cwm_data = self._cloudwatch.get_metric_statistics(**request)['Datapoints']
        if len(raw_cwm_data) == 0:
            logging.warning("Warning: No metrics called %s found" % metric_name)
            return

        # Process data: normalize to starting time, and sort.
        base_time = min(raw_cwm_data, key=lambda pt: pt['Timestamp'])['Timestamp']
        all_xy = []
        for pt in raw_cwm_data:
            y = pt['Average']
            x = (pt['Timestamp'] - base_time).total_seconds()
            all_xy.append([x, y])
        all_xy = sorted(all_xy, key=lambda x: x[0])

        # Store everything in _data to make a dataframe from
        for elapsed_seconds, value in all_xy:
            self._add_single_metric(elapsed_seconds, metric_name, value)


Instruction:
How would you implement a function in Python 3 that
stores a single metric in the _data dict which can be converted to a dataframe.

Response:
def _add_single_metric(self, timestamp, metric_name, value):
        """Store a single metric in the _data dict which can be
        converted to a dataframe.
        """
        # note that this method is built this way to make it possible to
        # support live-refreshing charts in Bokeh at some point in the future.
        self._data['timestamp'].append(timestamp)
        self._data['metric_name'].append(metric_name)
        self._data['value'].append(value)


Instruction:
Write a Python 3 script for
generating a unique name from a base string.

Response:
def name_from_base(base, max_length=63, short=False):
    """Append a timestamp to the provided string.

    This function assures that the total length of the resulting string is not
    longer than the specified max length, trimming the input parameter if necessary.

    Args:
        base (str): String used as prefix to generate the unique name.
        max_length (int): Maximum length for the resulting string.
        short (bool): Whether or not to use a truncated timestamp.

    Returns:
        str: Input parameter with appended timestamp.
    """
    timestamp = sagemaker_short_timestamp() if short else sagemaker_timestamp()
    trimmed_base = base[:max_length - len(timestamp) - 1]
    return '{}-{}'.format(trimmed_base, timestamp)


Instruction:
How would you explain what the following Python 3 function does
def base_name_from_image(image):
    m = re.match("^(.+/)?([^:/]+)(:[^:]+)?$", image)
    algo_name = m.group(2) if m else image
    return algo_name

Response: Extract the base name of the image to use as the algorithm name for the job.


Instruction:
Write a Python 3 script for
returning a timestamp with millisecond precision.

Response:
def sagemaker_timestamp():
    """Return a timestamp with millisecond precision."""
    moment = time.time()
    moment_ms = repr(moment).split('.')[1][:3]
    return time.strftime("%Y-%m-%d-%H-%M-%S-{}".format(moment_ms), time.gmtime(moment))


Instruction:
Write a Python 3 function that can
print the function name and arguments for debugging.

Response:
def debug(func):
    """Print the function name and arguments for debugging."""
    @wraps(func)
    def wrapper(*args, **kwargs):
        print("{} args: {} kwargs: {}".format(func.__name__, args, kwargs))
        return func(*args, **kwargs)

    return wrapper


Instruction:
Write a Python 3 script for
converting the input value to a string.

Response:
def to_str(value):
    """Convert the input to a string, unless it is a unicode string in Python 2.

    Unicode strings are supported as native strings in Python 3, but ``str()`` cannot be
    invoked on unicode strings in Python 2, so we need to check for that case when
    converting user-specified values to strings.

    Args:
        value: The value to convert to a string.

    Returns:
        str or unicode: The string representation of the value or the unicode string itself.
    """
    if sys.version_info.major < 3 and isinstance(value, six.string_types):
        return value
    return str(value)


Instruction:
Here you have a function in Python 3, explain what it does
def extract_name_from_job_arn(arn):
    slash_pos = arn.find('/')
    if slash_pos == -1:
        raise ValueError("Cannot parse invalid ARN: %s" % arn)
    return arn[(slash_pos + 1):]

Response:
Returns the name used in the API given a full ARN for a training job
    or hyperparameter tuning job.


Instruction:
Create a Python 3 function for
returning a string containing last modified time and the secondary training job status message.

Response:
def secondary_training_status_message(job_description, prev_description):
    """Returns a string contains last modified time and the secondary training job status message.

    Args:
        job_description: Returned response from DescribeTrainingJob call
        prev_description: Previous job description from DescribeTrainingJob call

    Returns:
        str: Job status string to be printed.

    """

    if job_description is None or job_description.get('SecondaryStatusTransitions') is None\
            or len(job_description.get('SecondaryStatusTransitions')) == 0:
        return ''

    prev_description_secondary_transitions = prev_description.get('SecondaryStatusTransitions')\
        if prev_description is not None else None
    prev_transitions_num = len(prev_description['SecondaryStatusTransitions'])\
        if prev_description_secondary_transitions is not None else 0
    current_transitions = job_description['SecondaryStatusTransitions']

    if len(current_transitions) == prev_transitions_num:
        # Secondary status is not changed but the message changed.
        transitions_to_print = current_transitions[-1:]
    else:
        # Secondary status is changed we need to print all the entries.
        transitions_to_print = current_transitions[prev_transitions_num - len(current_transitions):]

    status_strs = []
    for transition in transitions_to_print:
        message = transition['StatusMessage']
        time_str = datetime.utcfromtimestamp(
            time.mktime(job_description['LastModifiedTime'].timetuple())).strftime('%Y-%m-%d %H:%M:%S')
        status_strs.append('{} {} - {}'.format(time_str, transition['Status'], message))

    return '\n'.join(status_strs)


Instruction:
Write a Python 3 script to
download a folder from S3 to a local path.

Response:
def download_folder(bucket_name, prefix, target, sagemaker_session):
    """Download a folder from S3 to a local path

    Args:
        bucket_name (str): S3 bucket name
        prefix (str): S3 prefix within the bucket that will be downloaded. Can be a single file.
        target (str): destination path where the downloaded items will be placed
        sagemaker_session (:class:`sagemaker.session.Session`): a sagemaker session to interact with S3.
    """
    boto_session = sagemaker_session.boto_session

    s3 = boto_session.resource('s3')
    bucket = s3.Bucket(bucket_name)

    prefix = prefix.lstrip('/')

    # there is a chance that the prefix points to a file and not a 'directory' if that is the case
    # we should just download it.
    objects = list(bucket.objects.filter(Prefix=prefix))

    if len(objects) > 0 and objects[0].key == prefix and prefix[-1] != '/':
        s3.Object(bucket_name, prefix).download_file(os.path.join(target, os.path.basename(prefix)))
        return

    # the prefix points to an s3 'directory' download the whole thing
    for obj_sum in bucket.objects.filter(Prefix=prefix):
        # if obj_sum is a folder object skip it.
        if obj_sum.key != '' and obj_sum.key[-1] == '/':
            continue
        obj = s3.Object(obj_sum.bucket_name, obj_sum.key)
        s3_relative_path = obj_sum.key[len(prefix):].lstrip('/')
        file_path = os.path.join(target, s3_relative_path)

        try:
            os.makedirs(os.path.dirname(file_path))
        except OSError as exc:
            # EEXIST means the folder already exists, this is safe to skip
            # anything else will be raised.
            if exc.errno != errno.EEXIST:
                raise
        obj.download_file(file_path)


Instruction:
How would you explain what the following Python 3 function does
def create_tar_file(source_files, target=None):
    if target:
        filename = target
    else:
        _, filename = tempfile.mkstemp()

    with tarfile.open(filename, mode='w:gz') as t:
        for sf in source_files:
            # Add all files from the directory into the root of the directory structure of the tar
            t.add(sf, arcname=os.path.basename(sf))
    return filename

Response: Create a tar file containing all the source_files


Instruction:
How would you implement a function in Python 3 that
downloads a Single File from S3 into a local path.

Response:
def download_file(bucket_name, path, target, sagemaker_session):
    """Download a Single File from S3 into a local path

    Args:
        bucket_name (str): S3 bucket name
        path (str): file path within the bucket
        target (str): destination directory for the downloaded file.
        sagemaker_session (:class:`sagemaker.session.Session`): a sagemaker session to interact with S3.
    """
    path = path.lstrip('/')
    boto_session = sagemaker_session.boto_session

    s3 = boto_session.resource('s3')
    bucket = s3.Bucket(bucket_name)
    bucket.download_file(path, target)


Instruction:
How would you explain what the following Python 3 function does
def _sync_directories(from_directory, to_directory):
        if not os.path.exists(to_directory):
            os.mkdir(to_directory)
        for root, dirs, files in os.walk(from_directory):
            to_root = root.replace(from_directory, to_directory)
            for directory in dirs:
                to_child_dir = os.path.join(to_root, directory)
                if not os.path.exists(to_child_dir):
                    os.mkdir(to_child_dir)
            for fname in files:
                from_file = os.path.join(root, fname)
                to_file = os.path.join(to_root, fname)
                with open(from_file, 'rb') as a, open(to_file, 'wb') as b:
                    b.write(a.read())

Response: Syncs the contents of from_directory to to_directory by copying each file in from_directory with new contents.


Instruction:
Create a Python 3 function for
creating a TensorBoard process.

Response:
def create_tensorboard_process(self):
        """Create a TensorBoard process.

        Returns:
            tuple: A tuple containing:
                int: The port number.
                process: The TensorBoard process.

        Raises:
            OSError: If no ports between 6006 and 6105 are available for starting TensorBoard.
        """
        port = 6006

        for _ in range(100):
            p = subprocess.Popen(
                ["tensorboard", "--logdir", self.logdir, "--host", "localhost", "--port",
                 str(port)],
                stdout=subprocess.PIPE,
                stderr=subprocess.PIPE
            )
            self.event.wait(5)
            if p.poll():
                port += 1
            else:
                return port, p

        raise OSError(
            'No available ports to start TensorBoard. Attempted all ports between 6006 and 6105')


Instruction:
Can you create a Python 3 function that
trains a model using the input training dataset.

Response:
def fit(self, inputs=None, wait=True, logs=True, job_name=None, run_tensorboard_locally=False):
        """Train a model using the input training dataset.

        See :func:`~sagemaker.estimator.EstimatorBase.fit` for more details.

        Args:
            inputs (str or dict or sagemaker.session.s3_input): Information about the training data.
                This can be one of three types:

                * (str) - the S3 location where training data is saved.
                * (dict[str, str] or dict[str, sagemaker.session.s3_input]) - If using multiple channels for
                    training data, you can specify a dict mapping channel names
                    to strings or :func:`~sagemaker.session.s3_input` objects.
                * (sagemaker.session.s3_input) - channel configuration for S3 data sources that can provide
                    additional information as well as the path to the training dataset.
                    See :func:`sagemaker.session.s3_input` for full details.

            wait (bool): Whether the call should wait until the job completes (default: True).
            logs (bool): Whether to show the logs produced by the job.
                Only meaningful when wait is True (default: True).
            job_name (str): Training job name. If not specified, the estimator generates a default job name,
                based on the training image name and current timestamp.
            run_tensorboard_locally (bool): Whether to execute TensorBoard in a different process with
                downloaded checkpoint information (default: False). This is an experimental feature, and requires
                TensorBoard and AWS CLI to be installed. It terminates TensorBoard when execution ends.
        """

        def fit_super():
            super(TensorFlow, self).fit(inputs, wait, logs, job_name)

        if run_tensorboard_locally and wait is False:
            raise ValueError("Tensorboard is not supported with async fit")

        if self._script_mode_enabled() and run_tensorboard_locally:
            LOGGER.warning(_SCRIPT_MODE_TENSORBOARD_WARNING.format(self.model_dir))
            fit_super()
        elif run_tensorboard_locally:
            tensorboard = Tensorboard(self)
            tensorboard.validate_requirements()

            try:
                tensorboard.start()
                fit_super()
            finally:
                # sleep 20 secs for tensorboard start up if fit() quits instantly
                time.sleep(20)
                tensorboard.event.set()
                tensorboard.join()
        else:
            fit_super()


Instruction:
Can you generate the documentation for the following Python 3 function
def _prepare_init_params_from_job_description(cls, job_details, model_channel_name=None):
        init_params = super(TensorFlow, cls)._prepare_init_params_from_job_description(job_details,
                                                                                model_channel_name)

        # Move some of the tensorflow specific init params from hyperparameters into the main init params.
        for argument in ('checkpoint_path', 'training_steps', 'evaluation_steps', 'model_dir'):
            value = init_params['hyperparameters'].pop(argument, None)
            if value is not None:
                init_params[argument] = value

        image_name = init_params.pop('image')
        framework, py_version, tag, script_mode = fw.framework_name_from_image(image_name)
        if not framework:
            # If we were unable to parse the framework name from the image it is not one of our
            # officially supported images, in this case just add the image to the init params.
            init_params['image_name'] = image_name
            return init_params

        if script_mode:
            init_params['script_mode'] = True

        init_params['py_version'] = py_version

        # We switched image tagging scheme from regular image version (e.g. '1.0') to more expressive
        # containing framework version, device type and python version (e.g. '1.5-gpu-py2').
        # For backward compatibility map deprecated image tag '1.0' to a '1.4' framework version
        # otherwise extract framework version from the tag itself.
        init_params['framework_version'] = '1.4' if tag == '1.0' else fw.framework_version_from_tag(
            tag)

        training_job_name = init_params['base_job_name']
        if framework != cls.__framework_name__:
            raise ValueError("Training job: {} didn't use image for requested framework".format(
                training_job_name))

        return init_params

Response: Convert the job description to init params that can be handled by the class constructor.


Instruction:
Can you create a Python 3 function that
creates a SageMaker TensorFlowModel object that can be deployed to an Endpoint.

Response:
def create_model(self, model_server_workers=None, role=None,
                     vpc_config_override=VPC_CONFIG_DEFAULT, endpoint_type=None):
        """Create a SageMaker ``TensorFlowModel`` object that can be deployed to an ``Endpoint``.

        Args:
            role (str): The ``ExecutionRoleArn`` IAM Role ARN for the ``Model``, which is also used during
                transform jobs. If not specified, the role from the Estimator will be used.
            model_server_workers (int): Optional. The number of worker processes used by the inference server.
                If None, server will use one worker per vCPU.
            vpc_config_override (dict[str, list[str]]): Optional override for VpcConfig set on the model.
                Default: use subnets and security groups from this Estimator.
                * 'Subnets' (list[str]): List of subnet ids.
                * 'SecurityGroupIds' (list[str]): List of security group ids.
            endpoint_type: Optional. Selects the software stack used by the inference server.
                If  not specified, the model will be configured to use the default
                SageMaker model server. If 'tensorflow-serving', the model will be configured to
                use the SageMaker Tensorflow Serving container.

        Returns:
            sagemaker.tensorflow.model.TensorFlowModel: A SageMaker ``TensorFlowModel`` object.
                See :func:`~sagemaker.tensorflow.model.TensorFlowModel` for full details.
        """

        role = role or self.role
        if endpoint_type == 'tensorflow-serving' or self._script_mode_enabled():
            return self._create_tfs_model(role=role, vpc_config_override=vpc_config_override)

        return self._create_default_model(model_server_workers=model_server_workers, role=role,
                                          vpc_config_override=vpc_config_override)


Instruction:
Can you generate the documentation for the following Python 3 function
def hyperparameters(self):
        hyperparameters = super(TensorFlow, self).hyperparameters()

        self.checkpoint_path = self.checkpoint_path or self._default_s3_path('checkpoints')
        mpi_enabled = False

        if self._script_mode_enabled():
            additional_hyperparameters = {}

            if 'parameter_server' in self.distributions:
                ps_enabled = self.distributions['parameter_server'].get('enabled', False)
                additional_hyperparameters[self.LAUNCH_PS_ENV_NAME] = ps_enabled

            if 'mpi' in self.distributions:
                mpi_dict = self.distributions['mpi']
                mpi_enabled = mpi_dict.get('enabled', False)
                additional_hyperparameters[self.LAUNCH_MPI_ENV_NAME] = mpi_enabled
                additional_hyperparameters[self.MPI_NUM_PROCESSES_PER_HOST] = mpi_dict.get('processes_per_host', 1)
                additional_hyperparameters[self.MPI_CUSTOM_MPI_OPTIONS] = mpi_dict.get('custom_mpi_options', '')

            self.model_dir = self.model_dir or self._default_s3_path('model', mpi=mpi_enabled)
            additional_hyperparameters['model_dir'] = self.model_dir
        else:
            additional_hyperparameters = {'checkpoint_path': self.checkpoint_path,
                                          'training_steps': self.training_steps,
                                          'evaluation_steps': self.evaluation_steps,
                                          'sagemaker_requirements': self.requirements_file}

        hyperparameters.update(Framework._json_encode_hyperparameters(additional_hyperparameters))
        return hyperparameters

Response: Return hyperparameters used by your custom TensorFlow code during model training.


Instruction:
Explain what the following Python 3 code does
def from_dict(vpc_config, do_sanitize=False):
    if do_sanitize:
        vpc_config = sanitize(vpc_config)
    if vpc_config is None:
        return None, None
    return vpc_config[SUBNETS_KEY], vpc_config[SECURITY_GROUP_IDS_KEY]

Response: Extracts subnets and security group ids as lists from a VpcConfig dict containing SubnetIds and SecurityGroupId keys.


Instruction:
Here you have a function in Python 3, explain what it does
def sanitize(vpc_config):
    if vpc_config is None:
        return vpc_config
    elif type(vpc_config) is not dict:
        raise ValueError('vpc_config is not a dict: {}'.format(vpc_config))
    elif not vpc_config:
        raise ValueError('vpc_config is empty')

    subnets = vpc_config.get(SUBNETS_KEY)
    if subnets is None:
        raise ValueError('vpc_config is missing key: {}'.format(SUBNETS_KEY))
    if type(subnets) is not list:
        raise ValueError('vpc_config value for {} is not a list: {}'.format(SUBNETS_KEY, subnets))
    elif not subnets:
        raise ValueError('vpc_config value for {} is empty'.format(SUBNETS_KEY))

    security_group_ids = vpc_config.get(SECURITY_GROUP_IDS_KEY)
    if security_group_ids is None:
        raise ValueError('vpc_config is missing key: {}'.format(SECURITY_GROUP_IDS_KEY))
    if type(security_group_ids) is not list:
        raise ValueError('vpc_config value for {} is not a list: {}'.format(SECURITY_GROUP_IDS_KEY, security_group_ids))
    elif not security_group_ids:
        raise ValueError('vpc_config value for {} is empty'.format(SECURITY_GROUP_IDS_KEY))

    return to_dict(subnets, security_group_ids)

Response: Sanitize a VpcConfig dictionary to be used as a base for the cluster tree structure.


Instruction:
Create a Python 3 function to
return the SageMaker hyperparameters for training this KMeans Estimator

Response:
def hyperparameters(self):
        """Return the SageMaker hyperparameters for training this KMeans Estimator"""
        hp_dict = dict(force_dense='True')  # KMeans requires this hp to fit on Record objects
        hp_dict.update(super(KMeans, self).hyperparameters())
        return hp_dict


Instruction:
How would you explain what the following Python 3 function does
def create_identical_dataset_and_algorithm_tuner(parent, additional_parents=None, sagemaker_session=None):

    parent_tuner = HyperparameterTuner.attach(tuning_job_name=parent, sagemaker_session=sagemaker_session)
    return parent_tuner.identical_dataset_and_algorithm_tuner(additional_parents=additional_parents)

Response:
Creates a new HyperparameterTuner object which is a copy of the request fields from the provided parent to the new instance of
        IdenticalDataAndAlgorithm and additional_parents.


Instruction:
Given the following Python 3 function, write the documentation
def create_transfer_learning_tuner(parent, additional_parents=None, estimator=None, sagemaker_session=None):

    parent_tuner = HyperparameterTuner.attach(tuning_job_name=parent, sagemaker_session=sagemaker_session)
    return parent_tuner.transfer_learning_tuner(additional_parents=additional_parents,
                                                estimator=estimator)

Response: Creates a new instance of HyperparameterTuner that is a copy of the request fields from the provided parent to the new instance of HyperparameterTuner.


Instruction:
Explain what the following Python 3 code does
def from_job_desc(cls, warm_start_config):
        if not warm_start_config or \
                WARM_START_TYPE not in warm_start_config or \
                PARENT_HYPERPARAMETER_TUNING_JOBS not in warm_start_config:
            return None

        parents = []
        for parent in warm_start_config[PARENT_HYPERPARAMETER_TUNING_JOBS]:
            parents.append(parent[HYPERPARAMETER_TUNING_JOB_NAME])

        return cls(warm_start_type=WarmStartTypes(warm_start_config[WARM_START_TYPE]),
                   parents=parents)

Response: Creates an instance of this class from the given warm start configuration response.


Instruction:
How would you implement a function in Python 3 that
converts the self instance to the desired input request format.

Response:
def to_input_req(self):
        """Converts the ``self`` instance to the desired input request format.

        Returns:
            dict: Containing the "WarmStartType" and "ParentHyperParameterTuningJobs" as the first class fields.

        Examples:
            >>> warm_start_config = WarmStartConfig(warm_start_type=WarmStartTypes.TransferLearning,parents=["p1,p2"])
            >>> warm_start_config.to_input_req()
            {
                "WarmStartType":"TransferLearning",
                "ParentHyperParameterTuningJobs": [
                    {'HyperParameterTuningJobName': "p1"},
                    {'HyperParameterTuningJobName': "p2"},
                ]
            }
        """
        return {
            WARM_START_TYPE: self.type.value,
            PARENT_HYPERPARAMETER_TUNING_JOBS: [{HYPERPARAMETER_TUNING_JOB_NAME: parent} for parent in self.parents]
        }


Instruction:
Implement a Python 3 function for
starting a hyperparameter tuning job.

Response:
def fit(self, inputs=None, job_name=None, include_cls_metadata=False, **kwargs):
        """Start a hyperparameter tuning job.

        Args:
            inputs: Information about the training data. Please refer to the ``fit()`` method of
                the associated estimator, as this can take any of the following forms:

                * (str) - The S3 location where training data is saved.
                * (dict[str, str] or dict[str, sagemaker.session.s3_input]) - If using multiple channels for
                    training data, you can specify a dict mapping channel names
                    to strings or :func:`~sagemaker.session.s3_input` objects.
                * (sagemaker.session.s3_input) - Channel configuration for S3 data sources that can provide
                    additional information about the training dataset. See :func:`sagemaker.session.s3_input`
                    for full details.
                * (sagemaker.amazon.amazon_estimator.RecordSet) - A collection of
                    Amazon :class:~`Record` objects serialized and stored in S3.
                    For use with an estimator for an Amazon algorithm.
                * (list[sagemaker.amazon.amazon_estimator.RecordSet]) - A list of
                    :class:~`sagemaker.amazon.amazon_estimator.RecordSet` objects, where each instance is
                    a different channel of training data.

            job_name (str): Tuning job name. If not specified, the tuner generates a default job name,
                based on the training image name and current timestamp.
            include_cls_metadata (bool): Whether or not the hyperparameter tuning job should include
                information about the estimator class (default: False). This information is passed
                as a hyperparameter, so if the algorithm you are using cannot handle
                unknown hyperparameters (e.g. an Amazon SageMaker built-in algorithm that
                does not have a custom estimator in the Python SDK), then set
                ``include_cls_metadata`` to ``False``.
            **kwargs: Other arguments needed for training. Please refer to the ``fit()`` method of the associated
                estimator to see what other arguments are needed.
        """
        if isinstance(inputs, list) or isinstance(inputs, RecordSet):
            self.estimator._prepare_for_training(inputs, **kwargs)
        else:
            self.estimator._prepare_for_training(job_name)

        self._prepare_for_training(job_name=job_name, include_cls_metadata=include_cls_metadata)
        self.latest_tuning_job = _TuningJob.start_new(self, inputs)


Instruction:
Here you have a function in Python 3, explain what it does
def attach(cls, tuning_job_name, sagemaker_session=None, job_details=None, estimator_cls=None):
        sagemaker_session = sagemaker_session or Session()

        if job_details is None:
            job_details = sagemaker_session.sagemaker_client \
                .describe_hyper_parameter_tuning_job(HyperParameterTuningJobName=tuning_job_name)

        estimator_cls = cls._prepare_estimator_cls(estimator_cls, job_details['TrainingJobDefinition'])
        estimator = cls._prepare_estimator_from_job_description(estimator_cls, job_details['TrainingJobDefinition'],
                                                                sagemaker_session)
        init_params = cls._prepare_init_params_from_job_description(job_details)

        tuner = cls(estimator=estimator, **init_params)
        tuner.latest_tuning_job = _TuningJob(sagemaker_session=sagemaker_session, job_name=tuning_job_name)

        return tuner

Response: Attach to an existing hyperparameter tuning job.


Instruction:
Can you generate the documentation for the following Python 3 function
def deploy(self, initial_instance_count, instance_type, accelerator_type=None, endpoint_name=None, **kwargs):
        endpoint_name = endpoint_name or self.best_training_job()
        best_estimator = self.estimator.attach(self.best_training_job(),
                                               sagemaker_session=self.estimator.sagemaker_session)
        return best_estimator.deploy(initial_instance_count, instance_type,
                                     accelerator_type=accelerator_type,
                                     endpoint_name=endpoint_name, **kwargs)

Response: Deploy the best trained or user specified model to an Amazon SageMaker endpoint and return a SageMakerRealTimePredictor object.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def best_training_job(self):
        self._ensure_last_tuning_job()

        tuning_job_describe_result = \
            self.estimator.sagemaker_session.sagemaker_client.describe_hyper_parameter_tuning_job(
                HyperParameterTuningJobName=self.latest_tuning_job.name)

        try:
            return tuning_job_describe_result['BestTrainingJob']['TrainingJobName']
        except KeyError:
            raise Exception('Best training job not available for tuning job: {}'.format(self.latest_tuning_job.name))

Response: Return name of the best training job for the latest hyperparameter tuning job.


Instruction:
Implement a Python 3 function for
deleting an Amazon SageMaker endpoint.

Response:
def delete_endpoint(self, endpoint_name=None):
        """Delete an Amazon SageMaker endpoint.

        If an endpoint name is not specified, this defaults to looking for an endpoint that
        shares a name with the best training job for deletion.

        Args:
            endpoint_name (str): Name of the endpoint to delete
        """
        endpoint_name = endpoint_name or self.best_training_job()
        self.sagemaker_session.delete_endpoint(endpoint_name)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def hyperparameter_ranges(self):
        hyperparameter_ranges = dict()
        for range_type in ParameterRange.__all_types__:
            parameter_ranges = []
            for parameter_name, parameter in self._hyperparameter_ranges.items():
                if parameter is not None and parameter.__name__ == range_type:
                    # Categorical parameters needed to be serialized as JSON for our framework containers
                    if isinstance(parameter, CategoricalParameter) and isinstance(self.estimator, Framework):
                        tuning_range = parameter.as_json_range(parameter_name)
                    else:
                        tuning_range = parameter.as_tuning_range(parameter_name)
                    parameter_ranges.append(tuning_range)
            hyperparameter_ranges[range_type + 'ParameterRanges'] = parameter_ranges
        return hyperparameter_ranges

Response: Return the hyperparameter ranges in a dictionary to be used as part of a request for creating a hyperparameter tuning job.


Instruction:
Write a Python 3 script for
creating a new instance of a new HyperparameterTuner that will be used to create a new TransferLearningTuner instance.

Response:
def transfer_learning_tuner(self, additional_parents=None, estimator=None):
        """Creates a new ``HyperparameterTuner`` by copying the request fields from the provided parent to the new
        instance of ``HyperparameterTuner``. Followed by addition of warm start configuration with the type as
        "TransferLearning" and parents as the union of provided list of ``additional_parents`` and the ``self``.
        Also, training image in the new tuner's estimator is updated with the provided ``training_image``.

        Args:
            additional_parents (set{str}): Set of additional parents along with the self to be used in warm starting
            the transfer learning tuner.
            estimator (sagemaker.estimator.EstimatorBase): An estimator object that has been initialized with
                the desired configuration. There does not need to be a training job associated with this instance.

        Returns:
            sagemaker.tuner.HyperparameterTuner: ``HyperparameterTuner`` instance which can be used to launch transfer
            learning tuning job.

        Examples:
            >>> parent_tuner = HyperparameterTuner.attach(tuning_job_name="parent-job-1")
            >>> transfer_learning_tuner = parent_tuner.transfer_learning_tuner(additional_parents={"parent-job-2"})
            Later On:
            >>> transfer_learning_tuner.fit(inputs={})
        """

        return self._create_warm_start_tuner(additional_parents=additional_parents,
                                             warm_start_type=WarmStartTypes.TRANSFER_LEARNING,
                                             estimator=estimator)


Instruction:
Can you tell what is the following Python 3 function doing
def identical_dataset_and_algorithm_tuner(self, additional_parents=None):

        return self._create_warm_start_tuner(additional_parents=additional_parents,
                                             warm_start_type=WarmStartTypes.IDENTICAL_DATA_AND_ALGORITHM)

Response: Creates a new HyperparameterTuner instance which can be used to launch an identical dataset and algorithm tuner.


Instruction:
Make a summary of the following Python 3 code
def _create_warm_start_tuner(self, additional_parents, warm_start_type, estimator=None):
        all_parents = {self.latest_tuning_job.name}
        if additional_parents:
            all_parents = all_parents.union(additional_parents)

        return HyperparameterTuner(estimator=estimator if estimator else self.estimator,
                                   objective_metric_name=self.objective_metric_name,
                                   hyperparameter_ranges=self._hyperparameter_ranges,
                                   objective_type=self.objective_type,
                                   max_jobs=self.max_jobs,
                                   max_parallel_jobs=self.max_parallel_jobs,
                                   warm_start_config=WarmStartConfig(warm_start_type=warm_start_type,
                                                                     parents=all_parents))

Response: Create a new instance of type HyperparameterTuner with the request fields copied from self along with the warm start configuration.


Instruction:
Write a Python 3 script to
create a new Amazon SageMaker hyperparameter tuning job from the HyperparameterTuner.

Response:
def start_new(cls, tuner, inputs):
        """Create a new Amazon SageMaker hyperparameter tuning job from the HyperparameterTuner.

        Args:
            tuner (sagemaker.tuner.HyperparameterTuner): HyperparameterTuner object created by the user.
            inputs (str): Parameters used when called :meth:`~sagemaker.estimator.EstimatorBase.fit`.

        Returns:
            sagemaker.tuner._TuningJob: Constructed object that captures all information about the started job.
        """
        config = _Job._load_config(inputs, tuner.estimator)

        warm_start_config_req = None
        if tuner.warm_start_config:
            warm_start_config_req = tuner.warm_start_config.to_input_req()

        tuner_args = config.copy()

        tuner_args['job_name'] = tuner._current_job_name
        tuner_args['strategy'] = tuner.strategy
        tuner_args['objective_type'] = tuner.objective_type
        tuner_args['objective_metric_name'] = tuner.objective_metric_name
        tuner_args['max_jobs'] = tuner.max_jobs
        tuner_args['max_parallel_jobs'] = tuner.max_parallel_jobs
        tuner_args['parameter_ranges'] = tuner.hyperparameter_ranges()
        tuner_args['static_hyperparameters'] = tuner.static_hyperparameters
        tuner_args['input_mode'] = tuner.estimator.input_mode
        tuner_args['metric_definitions'] = tuner.metric_definitions
        tuner_args['tags'] = tuner.tags
        tuner_args['warm_start_config'] = warm_start_config_req
        tuner_args['early_stopping_type'] = tuner.early_stopping_type

        if isinstance(tuner.estimator, sagemaker.algorithm.AlgorithmEstimator):
            tuner_args['algorithm_arn'] = tuner.estimator.algorithm_arn
        else:
            tuner_args['image'] = tuner.estimator.train_image()

        tuner_args['enable_network_isolation'] = tuner.estimator.enable_network_isolation()
        tuner_args['encrypt_inter_container_traffic'] = \
            tuner.estimator.encrypt_inter_container_traffic

        tuner.estimator.sagemaker_session.tune(**tuner_args)

        return cls(tuner.sagemaker_session, tuner._current_job_name)


Instruction:
Here you have a function in Python 3, explain what it does
def some(arr):
    return functools.reduce(lambda x, y: x or (y is not None), arr, False)

Response: Return True iff there is an element a of arr such that a is not None


Instruction:
Make a summary of the following Python 3 code
def multi_stream_iter(client, log_group, streams, positions=None):
    positions = positions or {s: Position(timestamp=0, skip=0) for s in streams}
    event_iters = [log_stream(client, log_group, s, positions[s].timestamp, positions[s].skip) for s in streams]
    events = []
    for s in event_iters:
        if not s:
            events.append(None)
            continue
        try:
            events.append(next(s))
        except StopIteration:
            events.append(None)

    while some(events):
        i = argmin(events, lambda x: x['timestamp'] if x else 9999999999)
        yield (i, events[i])
        try:
            events[i] = next(event_iters[i])
        except StopIteration:
            events[i] = None

Response: Iterate over the available events coming from a single log group in a single log group.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def log_stream(client, log_group, stream_name, start_time=0, skip=0):

    next_token = None

    event_count = 1
    while event_count > 0:
        if next_token is not None:
            token_arg = {'nextToken': next_token}
        else:
            token_arg = {}

        response = client.get_log_events(logGroupName=log_group, logStreamName=stream_name, startTime=start_time,
                                         startFromHead=True, **token_arg)
        next_token = response['nextForwardToken']
        events = response['events']
        event_count = len(events)
        if event_count > skip:
            events = events[skip:]
            skip = 0
        else:
            skip = skip - event_count
            events = []
        for ev in events:
            yield ev

Response: A generator for log items in a single stream.


Instruction:
Can you generate the documentation for the following Python 3 function
def create_model(self, role=None, vpc_config_override=VPC_CONFIG_DEFAULT,
                     entry_point=None, source_dir=None, dependencies=None):
        """Create a SageMaker ``RLEstimatorModel`` object that can be deployed to an Endpoint.

        Args:
            role (str): The ``ExecutionRoleArn`` IAM Role ARN for the ``Model``, which is also used
                during transform jobs. If not specified, the role from the Estimator will be used.
            vpc_config_override (dict[str, list[str]]): Optional override for VpcConfig
                set on the model. Default: use subnets and security groups from this Estimator.

                * 'Subnets' (list[str]): List of subnet ids.
                * 'SecurityGroupIds' (list[str]): List of security group ids.

            entry_point (str): Path (absolute or relative) to the Python source file
                which should be executed as the entry point for MXNet hosting.
                This should be compatible with Python 3.5 (default: self.entry_point)
            source_dir (str): Path (absolute or relative) to a directory with any other training
                source code dependencies aside from tne entry point file (default: self.source_dir).
                Structure within this directory are preserved when hosting on Amazon SageMaker.
            dependencies (list[str]): A list of paths to directories (absolute or relative) with
                any additional libraries that will be exported to the container
                (default: self.dependencies). The library folders will be copied to SageMaker
                in the same folder where the entry_point is copied. If the ```source_dir``` points
                to S3, code will be uploaded and the S3 location will be used instead.

        Returns:
            sagemaker.model.FrameworkModel: Depending on input parameters returns
                one of the following:

                * sagemaker.model.FrameworkModel - in case image_name was specified
                    on the estimator;
                * sagemaker.mxnet.MXNetModel - if image_name wasn't specified and
                    MXNet was used as RL backend;
                * sagemaker.tensorflow.serving.Model - if image_name wasn't specified and
                    TensorFlow was used as RL backend.

        """
        base_args = dict(model_data=self.model_data,
                         role=role or self.role,
                         image=self.image_name,
                         name=self._current_job_name,
                         container_log_level=self.container_log_level,
                         sagemaker_session=self.sagemaker_session,
                         vpc_config=self.get_vpc_config(vpc_config_override))

        if not entry_point and (source_dir or dependencies):
            raise AttributeError('Please provide an `entry_point`.')

        entry_point = entry_point or self.entry_point
        source_dir = source_dir or self._model_source_dir()
        dependencies = dependencies or self.dependencies

        extended_args = dict(entry_point=entry_point,
                             source_dir=source_dir,
                             code_location=self.code_location,
                             dependencies=dependencies,
                             enable_cloudwatch_metrics=self.enable_cloudwatch_metrics)
        extended_args.update(base_args)

        if self.image_name:
            return FrameworkModel(**extended_args)

        if self.toolkit == RLToolkit.RAY.value:
            raise NotImplementedError(
                'Automatic deployment of Ray models is not currently available.'
                ' Train policy parameters are available in model checkpoints'
                ' in the TrainingJob output.'
            )

        if self.framework == RLFramework.TENSORFLOW.value:
            from sagemaker.tensorflow.serving import Model as tfsModel
            return tfsModel(framework_version=self.framework_version, **base_args)
        elif self.framework == RLFramework.MXNET.value:
            return MXNetModel(framework_version=self.framework_version, py_version=PYTHON_VERSION,
                              **extended_args)

Response: Create a SageMaker model object that can be deployed to an Endpoint.


Instruction:
Create a Python 3 function for
returning the URI of the Docker image to use for training.

Response:
def train_image(self):
        """Return the Docker image to use for training.

        The :meth:`~sagemaker.estimator.EstimatorBase.fit` method, which does the model training,
        calls this method to find the image to use for model training.

        Returns:
            str: The URI of the Docker image.
        """
        if self.image_name:
            return self.image_name
        else:
            return fw_utils.create_image_uri(self.sagemaker_session.boto_region_name,
                                             self._image_framework(),
                                             self.train_instance_type,
                                             self._image_version(),
                                             py_version=PYTHON_VERSION)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _prepare_init_params_from_job_description(cls, job_details, model_channel_name=None):
        init_params = super(RLEstimator, cls)\
            ._prepare_init_params_from_job_description(job_details, model_channel_name)

        image_name = init_params.pop('image')
        framework, _, tag, _ = fw_utils.framework_name_from_image(image_name)

        if not framework:
            # If we were unable to parse the framework name from the image it is not one of our
            # officially supported images, in this case just add the image to the init params.
            init_params['image_name'] = image_name
            return init_params

        toolkit, toolkit_version = cls._toolkit_and_version_from_tag(tag)

        if not cls._is_combination_supported(toolkit, toolkit_version, framework):
            training_job_name = init_params['base_job_name']
            raise ValueError(
                "Training job: {} didn't use image for requested framework".format(
                    training_job_name)
            )

        init_params['toolkit'] = RLToolkit(toolkit)
        init_params['toolkit_version'] = toolkit_version
        init_params['framework'] = RLFramework(framework)

        return init_params

Response: Convert the job description to init params that can be handled by the class constructor.


Instruction:
Can you tell what is the following Python 3 function doing
def hyperparameters(self):
        hyperparameters = super(RLEstimator, self).hyperparameters()

        additional_hyperparameters = {SAGEMAKER_OUTPUT_LOCATION: self.output_path,
                                      # TODO: can be applied to all other estimators
                                      SAGEMAKER_ESTIMATOR: SAGEMAKER_ESTIMATOR_VALUE}

        hyperparameters.update(Framework._json_encode_hyperparameters(additional_hyperparameters))
        return hyperparameters

Response: Return hyperparameters used by your custom TensorFlow code during model training.


Instruction:
Write a Python 3 function for
providing default metric definitions based on provided toolkit.

Response:
def default_metric_definitions(cls, toolkit):
        """Provides default metric definitions based on provided toolkit.

        Args:
            toolkit(sagemaker.rl.RLToolkit): RL Toolkit to be used for training.

        Returns:
            list: metric definitions
        """
        if toolkit is RLToolkit.COACH:
            return [
                {'Name': 'reward-training',
                 'Regex': '^Training>.*Total reward=(.*?),'},
                {'Name': 'reward-testing',
                 'Regex': '^Testing>.*Total reward=(.*?),'}
            ]
        elif toolkit is RLToolkit.RAY:
            float_regex = "[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?"  # noqa: W605, E501 pylint: disable=anomalous-backslash-in-string
            return [
                {'Name': 'episode_reward_mean',
                 'Regex': 'episode_reward_mean: (%s)' % float_regex},
                {'Name': 'episode_reward_max',
                 'Regex': 'episode_reward_max: (%s)' % float_regex}
            ]