Instruction:
Here you have a function in Python 3, explain what it does
def get_package_as_folder(dependency):

    venv_parent = tempfile.mkdtemp(
        prefix="pythonpackage-venv-"
    )
    try:
        # Create a venv to install into:
        try:
            if int(sys.version.partition(".")[0]) < 3:
                # Python 2.x has no venv.
                subprocess.check_output([
                    sys.executable,  # no venv conflict possible,
                                     # -> no need to use system python
                    "-m", "virtualenv",
                    "--python=" + _get_system_python_executable(),
                    os.path.join(venv_parent, 'venv')
                ], cwd=venv_parent)
            else:
                # On modern Python 3, use venv.
                subprocess.check_output([
                    _get_system_python_executable(), "-m", "venv",
                    os.path.join(venv_parent, 'venv')
                ], cwd=venv_parent)
        except subprocess.CalledProcessError as e:
            output = e.output.decode('utf-8', 'replace')
            raise ValueError(
                'venv creation unexpectedly ' +
                'failed. error output: ' + str(output)
            )
        venv_path = os.path.join(venv_parent, "venv")

        # Update pip and wheel in venv for latest feature support:
        try:
            filenotfounderror = FileNotFoundError
        except NameError:  # Python 2.
            filenotfounderror = OSError
        try:
            subprocess.check_output([
                os.path.join(venv_path, "bin", "pip"),
                "install", "-U", "pip", "wheel",
            ])
        except filenotfounderror:
            raise RuntimeError(
                "venv appears to be missing pip. "
                "did we fail to use a proper system python??\n"
                "system python path detected: {}\n"
                "os.environ['PATH']: {}".format(
                    _get_system_python_executable(),
                    os.environ.get("PATH", "")
                )
            )

        # Create download subfolder:
        os.mkdir(os.path.join(venv_path, "download"))

        # Write a requirements.txt with our package and download:
        with open(os.path.join(venv_path, "requirements.txt"),
                  "w", encoding="utf-8"
                 ) as f:
            def to_unicode(s):  # Needed for Python 2.
                try:
                    return s.decode("utf-8")
                except AttributeError:
                    return s
            f.write(to_unicode(transform_dep_for_pip(dependency)))
        try:
            subprocess.check_output(
                [
                    os.path.join(venv_path, "bin", "pip"),
                    "download", "--no-deps", "-r", "../requirements.txt",
                    "-d", os.path.join(venv_path, "download")
                ],
                stderr=subprocess.STDOUT,
                cwd=os.path.join(venv_path, "download")
            )
        except subprocess.CalledProcessError as e:
            raise RuntimeError("package download failed: " + str(e.output))

        if len(os.listdir(os.path.join(venv_path, "download"))) == 0:
            # No download. This can happen if the dependency has a condition
            # which prohibits install in our environment.
            # (the "package ; ... conditional ... " type of condition)
            return (None, None)

        # Get the result and make sure it's an extracted directory:
        result_folder_or_file = os.path.join(
            venv_path, "download",
            os.listdir(os.path.join(venv_path, "download"))[0]
        )
        dl_type = "source"
        if not os.path.isdir(result_folder_or_file):
            # Must be an archive.
            if result_folder_or_file.endswith((".zip", ".whl")):
                if result_folder_or_file.endswith(".whl"):
                    dl_type = "wheel"
                with zipfile.ZipFile(result_folder_or_file) as f:
                    f.extractall(os.path.join(venv_path,
                                              "download", "extracted"
                                             ))
                    result_folder_or_file = os.path.join(
                        venv_path, "download", "extracted"
                    )
            elif result_folder_or_file.find(".tar.") > 0:
                # Probably a tarball.
                with tarfile.open(result_folder_or_file) as f:
                    f.extractall(os.path.join(venv_path,
                                              "download", "extracted"
                                             ))
                    result_folder_or_file = os.path.join(
                        venv_path, "download", "extracted"
                    )
            else:
                raise RuntimeError(
                    "unknown archive or download " +
                    "type: " + str(result_folder_or_file)
                )

        # If the result is hidden away in an additional subfolder,
        # descend into it:
        while os.path.isdir(result_folder_or_file) and \
                len(os.listdir(result_folder_or_file)) == 1 and \
                os.path.isdir(os.path.join(
                    result_folder_or_file,
                    os.listdir(result_folder_or_file)[0]
                )):
            result_folder_or_file = os.path.join(
                result_folder_or_file,
                os.listdir(result_folder_or_file)[0]
            )

        # Copy result to new dedicated folder so we can throw away
        # our entire virtualenv nonsense after returning:
        result_path = tempfile.mkdtemp()
        shutil.rmtree(result_path)
        shutil.copytree(result_folder_or_file, result_path)
        return (dl_type, result_path)
    finally:
        shutil.rmtree(venv_parent)

Response: This function downloads the given package and extracts the raw contents into a folder.


Instruction:
How would you code a function in Python 3 to
parse a dependency reference into a folder path.

Response:
def parse_as_folder_reference(dep):
    """ See if a dependency reference refers to a folder path.
        If it does, return the folder path (which parses and
        resolves file:// urls in the process).
        If it doesn't, return None.
    """
    # Special case: pep508 urls
    if dep.find("@") > 0 and (
            (dep.find("@") < dep.find("/") or "/" not in dep) and
            (dep.find("@") < dep.find(":") or ":" not in dep)
            ):
        # This should be a 'pkgname @ https://...' style path, or
        # 'pkname @ /local/file/path'.
        return parse_as_folder_reference(dep.partition("@")[2].lstrip())

    # Check if this is either not an url, or a file URL:
    if dep.startswith(("/", "file://")) or (
            dep.find("/") > 0 and
            dep.find("://") < 0):
        if dep.startswith("file://"):
            dep = urlunquote(urlparse(dep).path)
        return dep
    return None


Instruction:
Explain what the following Python 3 code does
def _extract_info_from_package(dependency,
                               extract_type=None,
                               debug=False,
                               include_build_requirements=False
                               ):
    """ Internal function to extract metainfo from a package.
        Currently supported info types:

        - name
        - dependencies  (a list of dependencies)
    """
    output_folder = tempfile.mkdtemp(prefix="pythonpackage-metafolder-")
    try:
        extract_metainfo_files_from_package(
            dependency, output_folder, debug=debug
        )

        with open(os.path.join(output_folder, "METADATA"),
                  "r", encoding="utf-8"
                 ) as f:
            # Get metadata and cut away description (is after 2 linebreaks)
            metadata_entries = f.read().partition("\n\n")[0].splitlines()

        if extract_type == "name":
            name = None
            for meta_entry in metadata_entries:
                if meta_entry.lower().startswith("name:"):
                    return meta_entry.partition(":")[2].strip()
            if name is None:
                raise ValueError("failed to obtain package name")
            return name
        elif extract_type == "dependencies":
            requirements = []
            if os.path.exists(os.path.join(output_folder,
                                           'pyproject.toml')
                              ) and include_build_requirements:
                with open(os.path.join(output_folder, 'pyproject.toml')) as f:
                    build_sys = pytoml.load(f)['build-system']
                    if "requires" in build_sys:
                        requirements += build_sys["requires"]

            # Add requirements from metadata:
            requirements += [
                entry.rpartition("Requires-Dist:")[2].strip()
                for entry in metadata_entries
                if entry.startswith("Requires-Dist")
            ]

            return list(set(requirements))  # remove duplicates
    finally:
        shutil.rmtree(output_folder)

Response: Internal function to extract metainfo from a package.


Instruction:
Implement a Python 3 function for
getting the dependencies of a package.

Response:
def get_package_dependencies(package,
                             recursive=False,
                             verbose=False,
                             include_build_requirements=False):
    """ Obtain the dependencies from a package. Please note this
        function is possibly SLOW, especially if you enable
        the recursive mode.
    """
    packages_processed = set()
    package_queue = [package]
    reqs = set()
    reqs_as_names = set()
    while len(package_queue) > 0:
        current_queue = package_queue
        package_queue = []
        for package_dep in current_queue:
            new_reqs = set()
            if verbose:
                print("get_package_dependencies: resolving dependecy "
                      "to package name: ".format(package_dep))
            package = get_package_name(package_dep)
            if package.lower() in packages_processed:
                continue
            if verbose:
                print("get_package_dependencies: "
                      "processing package: {}".format(package))
                print("get_package_dependencies: "
                      "Packages seen so far: {}".format(
                          packages_processed
                      ))
            packages_processed.add(package.lower())

            # Use our regular folder processing to examine:
            new_reqs = new_reqs.union(_extract_info_from_package(
                package_dep, extract_type="dependencies",
                debug=verbose,
                include_build_requirements=include_build_requirements,
            ))

            # Process new requirements:
            if verbose:
                print('get_package_dependencies: collected '
                      "deps of '{}': {}".format(
                          package_dep, str(new_reqs),
                      ))
            for new_req in new_reqs:
                try:
                    req_name = get_package_name(new_req)
                except ValueError as e:
                    if new_req.find(";") >= 0:
                        # Conditional dep where condition isn't met?
                        # --> ignore it
                        continue
                    if verbose:
                        print("get_package_dependencies: " +
                              "unexpected failure to get name " +
                              "of '" + str(new_req) + "': " +
                              str(e))
                    raise RuntimeError(
                        "failed to get " +
                        "name of dependency: " + str(e)
                    )
                if req_name.lower() in reqs_as_names:
                    continue
                if req_name.lower() not in packages_processed:
                    package_queue.append(new_req)
                reqs.add(new_req)
                reqs_as_names.add(req_name.lower())

            # Bail out here if we're not scanning recursively:
            if not recursive:
                package_queue[:] = []  # wipe queue
                break
    if verbose:
        print("get_package_dependencies: returning result: {}".format(reqs))
    return reqs


Instruction:
Write a Python 3 function for
getting the dependency names of a package in the given folder.

Response:
def get_dep_names_of_package(
        package,
        keep_version_pins=False,
        recursive=False,
        verbose=False,
        include_build_requirements=False
        ):
    """ Gets the dependencies from the package in the given folder,
        then attempts to deduce the actual package name resulting
        from each dependency line, stripping away everything else.
    """

    # First, obtain the dependencies:
    dependencies = get_package_dependencies(
        package, recursive=recursive, verbose=verbose,
        include_build_requirements=include_build_requirements,
    )
    if verbose:
        print("get_dep_names_of_package_folder: " +
              "processing dependency list to names: " +
              str(dependencies))

    # Transform dependencies to their stripped down names:
    # (they can still have version pins/restrictions, conditionals, ...)
    dependency_names = set()
    for dep in dependencies:
        # If we are supposed to keep exact version pins, extract first:
        pin_to_append = ""
        if keep_version_pins and "(==" in dep and dep.endswith(")"):
            # This is a dependency of the format: 'pkg (==1.0)'
            pin_to_append = "==" + dep.rpartition("==")[2][:-1]
        elif keep_version_pins and "==" in dep and not dep.endswith(")"):
            # This is a dependency of the format: 'pkg==1.0'
            pin_to_append = "==" + dep.rpartition("==")[2]
        # Now get true (and e.g. case-corrected) dependency name:
        dep_name = get_package_name(dep) + pin_to_append
        dependency_names.add(dep_name)
    return dependency_names


Instruction:
Here you have a function in Python 3, explain what it does
def run_pymodules_install(ctx, modules, project_dir, ignore_setup_py=False):

    info('*** PYTHON PACKAGE / PROJECT INSTALL STAGE ***')
    modules = list(filter(ctx.not_has_package, modules))

    # Bail out if no python deps and no setup.py to process:
    if not modules and (
            ignore_setup_py or
            not project_has_setup_py(project_dir)
            ):
        info('No Python modules and no setup.py to process, skipping')
        return

    # Output messages about what we're going to do:
    if modules:
        info('The requirements ({}) don\'t have recipes, attempting to '
             'install them with pip'.format(', '.join(modules)))
        info('If this fails, it may mean that the module has compiled '
             'components and needs a recipe.')
    if project_has_setup_py(project_dir) and not ignore_setup_py:
        info('Will process project install, if it fails then the '
             'project may not be compatible for Android install.')

    venv = sh.Command(ctx.virtualenv)
    with current_directory(join(ctx.build_dir)):
        shprint(venv,
                '--python=python{}'.format(
                    ctx.python_recipe.major_minor_version_string.
                    partition(".")[0]
                    ),
                'venv'
               )

        # Prepare base environment and upgrade pip:
        base_env = copy.copy(os.environ)
        base_env["PYTHONPATH"] = ctx.get_site_packages_dir()
        info('Upgrade pip to latest version')
        shprint(sh.bash, '-c', (
            "source venv/bin/activate && pip install -U pip"
        ), _env=copy.copy(base_env))

        # Install Cython in case modules need it to build:
        info('Install Cython in case one of the modules needs it to build')
        shprint(sh.bash, '-c', (
            "venv/bin/pip install Cython"
        ), _env=copy.copy(base_env))

        # Get environment variables for build (with CC/compiler set):
        standard_recipe = CythonRecipe()
        standard_recipe.ctx = ctx
        # (note: following line enables explicit -lpython... linker options)
        standard_recipe.call_hostpython_via_targetpython = False
        recipe_env = standard_recipe.get_recipe_env(ctx.archs[0])
        env = copy.copy(base_env)
        env.update(recipe_env)

        # Make sure our build package dir is available, and the virtualenv
        # site packages come FIRST (so the proper pip version is used):
        env["PYTHONPATH"] += ":" + ctx.get_site_packages_dir()
        env["PYTHONPATH"] = os.path.abspath(join(
            ctx.build_dir, "venv", "lib",
            "python" + ctx.python_recipe.major_minor_version_string,
            "site-packages")) + ":" + env["PYTHONPATH"]

        # Install the manually specified requirements first:
        if not modules:
            info('There are no Python modules to install, skipping')
        else:
            info('Creating a requirements.txt file for the Python modules')
            with open('requirements.txt', 'w') as fileh:
                for module in modules:
                    key = 'VERSION_' + module
                    if key in environ:
                        line = '{}=={}\n'.format(module, environ[key])
                    else:
                        line = '{}\n'.format(module)
                    fileh.write(line)

            info('Installing Python modules with pip')
            info('IF THIS FAILS, THE MODULES MAY NEED A RECIPE. '
                 'A reason for this is often modules compiling '
                 'native code that is unaware of Android cross-compilation '
                 'and does not work without additional '
                 'changes / workarounds.')

            shprint(sh.bash, '-c', (
                "venv/bin/pip " +
                "install -v --target '{0}' --no-deps -r requirements.txt"
            ).format(ctx.get_site_packages_dir().replace("'", "'\"'\"'")),
                    _env=copy.copy(env))

        # Afterwards, run setup.py if present:
        if project_has_setup_py(project_dir) and not ignore_setup_py:
            with current_directory(project_dir):
                info('got setup.py or similar, running project install. ' +
                     '(disable this behavior with --ignore-setup-py)')

                # Compute & output the constraints we will use:
                info('Contents that will be used for constraints.txt:')
                constraints = subprocess.check_output([
                    join(
                        ctx.build_dir, "venv", "bin", "pip"
                    ),
                    "freeze"
                ], env=copy.copy(env))
                try:
                    constraints = constraints.decode("utf-8", "replace")
                except AttributeError:
                    pass
                info(constraints)

                # Make sure all packages found are fixed in version
                # by writing a constraint file, to avoid recipes being
                # upgraded & reinstalled:
                with open('constraints.txt', 'wb') as fileh:
                    fileh.write(constraints.encode("utf-8", "replace"))

                info('Populating venv\'s site-packages with '
                     'ctx.get_site_packages_dir()...')

                # Copy dist contents into site-packages for discovery.
                # Why this is needed:
                # --target is somewhat evil and messes with discovery of
                # packages in PYTHONPATH if that also includes the target
                # folder. So we need to use the regular virtualenv
                # site-packages folder instead.
                # Reference:
                # https://github.com/pypa/pip/issues/6223
                ctx_site_packages_dir = os.path.normpath(
                    os.path.abspath(ctx.get_site_packages_dir())
                )
                venv_site_packages_dir = os.path.normpath(os.path.join(
                    ctx.build_dir, "venv", "lib", [
                        f for f in os.listdir(os.path.join(
                            ctx.build_dir, "venv", "lib"
                        )) if f.startswith("python")
                    ][0], "site-packages"
                ))
                copied_over_contents = []
                for f in os.listdir(ctx_site_packages_dir):
                    full_path = os.path.join(ctx_site_packages_dir, f)
                    if not os.path.exists(os.path.join(
                                venv_site_packages_dir, f
                            )):
                        if os.path.isdir(full_path):
                            shutil.copytree(full_path, os.path.join(
                                venv_site_packages_dir, f
                            ))
                        else:
                            shutil.copy2(full_path, os.path.join(
                                venv_site_packages_dir, f
                            ))
                        copied_over_contents.append(f)

                # Get listing of virtualenv's site-packages, to see the
                # newly added things afterwards & copy them back into
                # the distribution folder / build context site-packages:
                previous_venv_contents = os.listdir(venv_site_packages_dir)

                # Actually run setup.py:
                info('Launching package install...')
                shprint(sh.bash, '-c', (
                    "'" + join(
                        ctx.build_dir, "venv", "bin", "pip"
                    ).replace("'", "'\"'\"'") + "' " +
                    "install -c constraints.txt -v ."
                ).format(ctx.get_site_packages_dir().replace("'", "'\"'\"'")),
                        _env=copy.copy(env))

                # Go over all new additions and copy them back:
                info('Copying additions resulting from setup.py back ' +
                     'into ctx.get_site_packages_dir()...')
                new_venv_additions = []
                for f in (set(os.listdir(venv_site_packages_dir)) -
                          set(previous_venv_contents)):
                    new_venv_additions.append(f)
                    full_path = os.path.join(venv_site_packages_dir, f)
                    if os.path.isdir(full_path):
                        shutil.copytree(full_path, os.path.join(
                            ctx_site_packages_dir, f
                        ))
                    else:
                        shutil.copy2(full_path, os.path.join(
                            ctx_site_packages_dir, f
                        ))

                # Undo all the changes we did to the venv-site packages:
                info('Reverting additions to virtualenv\'s site-packages...')
                for f in set(copied_over_contents + new_venv_additions):
                    full_path = os.path.join(venv_site_packages_dir, f)
                    if os.path.isdir(full_path):
                        shutil.rmtree(full_path)
                    else:
                        os.remove(full_path)
        elif not ignore_setup_py:
            info("No setup.py found in project directory: " +
                 str(project_dir)
                )

        # Strip object files after potential Cython or native code builds:
        standard_recipe.strip_object_files(ctx.archs[0], env,
                                           build_dir=ctx.build_dir)

Response: This function will process all python modules and install them via pip.


Instruction:
Can you create a Python 3 function that
calculates all the storage and build dirs and makes sure the directories exist where necessary.

Response:
def setup_dirs(self, storage_dir):
        '''Calculates all the storage and build dirs, and makes sure
        the directories exist where necessary.'''
        self.storage_dir = expanduser(storage_dir)
        if ' ' in self.storage_dir:
            raise ValueError('storage dir path cannot contain spaces, please '
                             'specify a path with --storage-dir')
        self.build_dir = join(self.storage_dir, 'build')
        self.dist_dir = join(self.storage_dir, 'dists')


Instruction:
Write a Python 3 function that can
prepare the build environment for the Android SDK and the Android API.

Response:
def prepare_build_environment(self,
                                  user_sdk_dir,
                                  user_ndk_dir,
                                  user_android_api,
                                  user_ndk_api):
        '''Checks that build dependencies exist and sets internal variables
        for the Android SDK etc.

        ..warning:: This *must* be called before trying any build stuff

        '''

        self.ensure_dirs()

        if self._build_env_prepared:
            return

        ok = True

        # Work out where the Android SDK is
        sdk_dir = None
        if user_sdk_dir:
            sdk_dir = user_sdk_dir
        # This is the old P4A-specific var
        if sdk_dir is None:
            sdk_dir = environ.get('ANDROIDSDK', None)
        # This seems used more conventionally
        if sdk_dir is None:
            sdk_dir = environ.get('ANDROID_HOME', None)
        # Checks in the buildozer SDK dir, useful for debug tests of p4a
        if sdk_dir is None:
            possible_dirs = glob.glob(expanduser(join(
                '~', '.buildozer', 'android', 'platform', 'android-sdk-*')))
            possible_dirs = [d for d in possible_dirs if not
                             (d.endswith('.bz2') or d.endswith('.gz'))]
            if possible_dirs:
                info('Found possible SDK dirs in buildozer dir: {}'.format(
                    ', '.join([d.split(os.sep)[-1] for d in possible_dirs])))
                info('Will attempt to use SDK at {}'.format(possible_dirs[0]))
                warning('This SDK lookup is intended for debug only, if you '
                        'use python-for-android much you should probably '
                        'maintain your own SDK download.')
                sdk_dir = possible_dirs[0]
        if sdk_dir is None:
            raise BuildInterruptingException('Android SDK dir was not specified, exiting.')
        self.sdk_dir = realpath(sdk_dir)

        # Check what Android API we're using
        android_api = None
        if user_android_api:
            android_api = user_android_api
            info('Getting Android API version from user argument: {}'.format(android_api))
        elif 'ANDROIDAPI' in environ:
            android_api = environ['ANDROIDAPI']
            info('Found Android API target in $ANDROIDAPI: {}'.format(android_api))
        else:
            info('Android API target was not set manually, using '
                 'the default of {}'.format(RECOMMENDED_TARGET_API))
            android_api = RECOMMENDED_TARGET_API
        android_api = int(android_api)
        self.android_api = android_api

        check_target_api(android_api, self.archs[0].arch)

        if exists(join(sdk_dir, 'tools', 'bin', 'avdmanager')):
            avdmanager = sh.Command(join(sdk_dir, 'tools', 'bin', 'avdmanager'))
            targets = avdmanager('list', 'target').stdout.decode('utf-8').split('\n')
        elif exists(join(sdk_dir, 'tools', 'android')):
            android = sh.Command(join(sdk_dir, 'tools', 'android'))
            targets = android('list').stdout.decode('utf-8').split('\n')
        else:
            raise BuildInterruptingException(
                'Could not find `android` or `sdkmanager` binaries in Android SDK',
                instructions='Make sure the path to the Android SDK is correct')
        apis = [s for s in targets if re.match(r'^ *API level: ', s)]
        apis = [re.findall(r'[0-9]+', s) for s in apis]
        apis = [int(s[0]) for s in apis if s]
        info('Available Android APIs are ({})'.format(
            ', '.join(map(str, apis))))
        if android_api in apis:
            info(('Requested API target {} is available, '
                  'continuing.').format(android_api))
        else:
            raise BuildInterruptingException(
                ('Requested API target {} is not available, install '
                 'it with the SDK android tool.').format(android_api))

        # Find the Android NDK
        # Could also use ANDROID_NDK, but doesn't look like many tools use this
        ndk_dir = None
        if user_ndk_dir:
            ndk_dir = user_ndk_dir
            info('Getting NDK dir from from user argument')
        if ndk_dir is None:  # The old P4A-specific dir
            ndk_dir = environ.get('ANDROIDNDK', None)
            if ndk_dir is not None:
                info('Found NDK dir in $ANDROIDNDK: {}'.format(ndk_dir))
        if ndk_dir is None:  # Apparently the most common convention
            ndk_dir = environ.get('NDK_HOME', None)
            if ndk_dir is not None:
                info('Found NDK dir in $NDK_HOME: {}'.format(ndk_dir))
        if ndk_dir is None:  # Another convention (with maven?)
            ndk_dir = environ.get('ANDROID_NDK_HOME', None)
            if ndk_dir is not None:
                info('Found NDK dir in $ANDROID_NDK_HOME: {}'.format(ndk_dir))
        if ndk_dir is None:  # Checks in the buildozer NDK dir, useful
            #                # for debug tests of p4a
            possible_dirs = glob.glob(expanduser(join(
                '~', '.buildozer', 'android', 'platform', 'android-ndk-r*')))
            if possible_dirs:
                info('Found possible NDK dirs in buildozer dir: {}'.format(
                    ', '.join([d.split(os.sep)[-1] for d in possible_dirs])))
                info('Will attempt to use NDK at {}'.format(possible_dirs[0]))
                warning('This NDK lookup is intended for debug only, if you '
                        'use python-for-android much you should probably '
                        'maintain your own NDK download.')
                ndk_dir = possible_dirs[0]
        if ndk_dir is None:
            raise BuildInterruptingException('Android NDK dir was not specified')
        self.ndk_dir = realpath(ndk_dir)

        check_ndk_version(ndk_dir)

        ndk_api = None
        if user_ndk_api:
            ndk_api = user_ndk_api
            info('Getting NDK API version (i.e. minimum supported API) from user argument')
        elif 'NDKAPI' in environ:
            ndk_api = environ.get('NDKAPI', None)
            info('Found Android API target in $NDKAPI')
        else:
            ndk_api = min(self.android_api, RECOMMENDED_NDK_API)
            warning('NDK API target was not set manually, using '
                    'the default of {} = min(android-api={}, default ndk-api={})'.format(
                        ndk_api, self.android_api, RECOMMENDED_NDK_API))
        ndk_api = int(ndk_api)
        self.ndk_api = ndk_api

        check_ndk_api(ndk_api, self.android_api)

        virtualenv = get_virtualenv_executable()
        if virtualenv is None:
            raise IOError('Couldn\'t find a virtualenv executable, '
                          'you must install this to use p4a.')
        self.virtualenv = virtualenv
        info('Found virtualenv at {}'.format(virtualenv))

        # path to some tools
        self.ccache = sh.which("ccache")
        if not self.ccache:
            info('ccache is missing, the build will not be optimized in the '
                 'future.')
        for cython_fn in ("cython", "cython3", "cython2", "cython-2.7"):
            cython = sh.which(cython_fn)
            if cython:
                self.cython = cython
                break
        else:
            raise BuildInterruptingException('No cython binary found.')
        if not self.cython:
            ok = False
            warning("Missing requirement: cython is not installed")

        # This would need to be changed if supporting multiarch APKs
        arch = self.archs[0]
        platform_dir = arch.platform_dir
        toolchain_prefix = arch.toolchain_prefix
        toolchain_version = None
        self.ndk_platform = join(
            self.ndk_dir,
            'platforms',
            'android-{}'.format(self.ndk_api),
            platform_dir)
        if not exists(self.ndk_platform):
            warning('ndk_platform doesn\'t exist: {}'.format(
                self.ndk_platform))
            ok = False

        py_platform = sys.platform
        if py_platform in ['linux2', 'linux3']:
            py_platform = 'linux'

        toolchain_versions = []
        toolchain_path = join(self.ndk_dir, 'toolchains')
        if isdir(toolchain_path):
            toolchain_contents = glob.glob('{}/{}-*'.format(toolchain_path,
                                                            toolchain_prefix))
            toolchain_versions = [split(path)[-1][len(toolchain_prefix) + 1:]
                                  for path in toolchain_contents]
        else:
            warning('Could not find toolchain subdirectory!')
            ok = False
        toolchain_versions.sort()

        toolchain_versions_gcc = []
        for toolchain_version in toolchain_versions:
            if toolchain_version[0].isdigit():
                # GCC toolchains begin with a number
                toolchain_versions_gcc.append(toolchain_version)

        if toolchain_versions:
            info('Found the following toolchain versions: {}'.format(
                toolchain_versions))
            info('Picking the latest gcc toolchain, here {}'.format(
                toolchain_versions_gcc[-1]))
            toolchain_version = toolchain_versions_gcc[-1]
        else:
            warning('Could not find any toolchain for {}!'.format(
                toolchain_prefix))
            ok = False

        self.toolchain_prefix = toolchain_prefix
        self.toolchain_version = toolchain_version
        # Modify the path so that sh finds modules appropriately
        environ['PATH'] = (
            '{ndk_dir}/toolchains/{toolchain_prefix}-{toolchain_version}/'
            'prebuilt/{py_platform}-x86/bin/:{ndk_dir}/toolchains/'
            '{toolchain_prefix}-{toolchain_version}/prebuilt/'
            '{py_platform}-x86_64/bin/:{ndk_dir}:{sdk_dir}/'
            'tools:{path}').format(
                sdk_dir=self.sdk_dir, ndk_dir=self.ndk_dir,
                toolchain_prefix=toolchain_prefix,
                toolchain_version=toolchain_version,
                py_platform=py_platform, path=environ.get('PATH'))

        for executable in ("pkg-config", "autoconf", "automake", "libtoolize",
                           "tar", "bzip2", "unzip", "make", "gcc", "g++"):
            if not sh.which(executable):
                warning("Missing executable: {} is not installed".format(
                    executable))

        if not ok:
            raise BuildInterruptingException(
                'python-for-android cannot continue due to the missing executables above')


Instruction:
Create a Python 3 function to
return the location of site - packages in the python - install build dir.

Response:
def get_site_packages_dir(self, arch=None):
        '''Returns the location of site-packages in the python-install build
        dir.
        '''
        if self.python_recipe.name == 'python2legacy':
            return join(self.get_python_install_dir(),
                        'lib', 'python2.7', 'site-packages')
        return self.get_python_install_dir()


Instruction:
Can you generate the documentation for the following Python 3 function
def get_libs_dir(self, arch):
        '''The libs dir for a given arch.'''
        ensure_dir(join(self.libs_dir, arch))
        return join(self.libs_dir, arch)

Response: The libs dir for a given arch.


Instruction:
Can you create a Python 3 function that
makes limited length string in form : the string is very lo... and 15 more

Response:
def shorten_string(string, max_width):
    ''' make limited length string in form:
      "the string is very lo...(and 15 more)"
    '''
    string_len = len(string)
    if string_len <= max_width:
        return string
    visible = max_width - 16 - int(log10(string_len))
    # expected suffix len "...(and XXXXX more)"
    if not isinstance(string, unistr):
        visstring = unistr(string[:visible], errors='ignore')
    else:
        visstring = string[:visible]
    return u''.join((visstring, u'...(and ',
                     unistr(string_len - visible), u' more)'))


Instruction:
Create a Python 3 function to
run the command while Formula logging the output.

Response:
def shprint(command, *args, **kwargs):
    '''Runs the command (which should be an sh.Command instance), while
    logging the output.'''
    kwargs["_iter"] = True
    kwargs["_out_bufsize"] = 1
    kwargs["_err_to_out"] = True
    kwargs["_bg"] = True
    is_critical = kwargs.pop('_critical', False)
    tail_n = kwargs.pop('_tail', None)
    full_debug = False
    if "P4A_FULL_DEBUG" in os.environ:
        tail_n = 0
        full_debug = True
    filter_in = kwargs.pop('_filter', None)
    filter_out = kwargs.pop('_filterout', None)
    if len(logger.handlers) > 1:
        logger.removeHandler(logger.handlers[1])
    columns = get_console_width()
    command_path = str(command).split('/')
    command_string = command_path[-1]
    string = ' '.join(['{}->{} running'.format(Out_Fore.LIGHTBLACK_EX,
                                               Out_Style.RESET_ALL),
                       command_string] + list(args))

    # If logging is not in DEBUG mode, trim the command if necessary
    if logger.level > logging.DEBUG:
        logger.info('{}{}'.format(shorten_string(string, columns - 12),
                                  Err_Style.RESET_ALL))
    else:
        logger.debug('{}{}'.format(string, Err_Style.RESET_ALL))

    need_closing_newline = False
    try:
        msg_hdr = '           working: '
        msg_width = columns - len(msg_hdr) - 1
        output = command(*args, **kwargs)
        for line in output:
            if isinstance(line, bytes):
                line = line.decode('utf-8', errors='replace')
            if logger.level > logging.DEBUG:
                if full_debug:
                    stdout.write(line)
                    stdout.flush()
                    continue
                msg = line.replace(
                    '\n', ' ').replace(
                        '\t', ' ').replace(
                            '\b', ' ').rstrip()
                if msg:
                    if "CI" not in os.environ:
                        stdout.write(u'{}\r{}{:<{width}}'.format(
                            Err_Style.RESET_ALL, msg_hdr,
                            shorten_string(msg, msg_width), width=msg_width))
                        stdout.flush()
                        need_closing_newline = True
            else:
                logger.debug(''.join(['\t', line.rstrip()]))
        if need_closing_newline:
            stdout.write('{}\r{:>{width}}\r'.format(
                Err_Style.RESET_ALL, ' ', width=(columns - 1)))
            stdout.flush()
    except sh.ErrorReturnCode as err:
        if need_closing_newline:
            stdout.write('{}\r{:>{width}}\r'.format(
                Err_Style.RESET_ALL, ' ', width=(columns - 1)))
            stdout.flush()
        if tail_n is not None or filter_in or filter_out:
            def printtail(out, name, forecolor, tail_n=0,
                          re_filter_in=None, re_filter_out=None):
                lines = out.splitlines()
                if re_filter_in is not None:
                    lines = [l for l in lines if re_filter_in.search(l)]
                if re_filter_out is not None:
                    lines = [l for l in lines if not re_filter_out.search(l)]
                if tail_n == 0 or len(lines) <= tail_n:
                    info('{}:\n{}\t{}{}'.format(
                        name, forecolor, '\t\n'.join(lines), Out_Fore.RESET))
                else:
                    info('{} (last {} lines of {}):\n{}\t{}{}'.format(
                        name, tail_n, len(lines),
                        forecolor, '\t\n'.join([s for s in lines[-tail_n:]]),
                        Out_Fore.RESET))
            printtail(err.stdout.decode('utf-8'), 'STDOUT', Out_Fore.YELLOW, tail_n,
                      re.compile(filter_in) if filter_in else None,
                      re.compile(filter_out) if filter_out else None)
            printtail(err.stderr.decode('utf-8'), 'STDERR', Err_Fore.RED)
        if is_critical:
            env = kwargs.get("env")
            if env is not None:
                info("{}ENV:{}\n{}\n".format(
                    Err_Fore.YELLOW, Err_Fore.RESET, "\n".join(
                        "set {}={}".format(n, v) for n, v in env.items())))
            info("{}COMMAND:{}\ncd {} && {} {}\n".format(
                Err_Fore.YELLOW, Err_Fore.RESET, os.getcwd(), command,
                ' '.join(args)))
            warning("{}ERROR: {} failed!{}".format(
                Err_Fore.RED, command, Err_Fore.RESET))
            exit(1)
        else:
            raise

    return output


Instruction:
Write a Python 3 script to
add an argument to the icalEntry object

Response:
def add_argument(self, *args, **kwargs):
        """
        add_argument(dest, ..., name=value, ...)
        add_argument(option_string, option_string, ..., name=value, ...)
        """

        # if no positional args are supplied or only one is supplied and
        # it doesn't look like an option string, parse a positional
        # argument
        chars = self.prefix_chars
        if not args or len(args) == 1 and args[0][0] not in chars:
            if args and 'dest' in kwargs:
                raise ValueError('dest supplied twice for positional argument')
            kwargs = self._get_positional_kwargs(*args, **kwargs)

        # otherwise, we're adding an optional argument
        else:
            kwargs = self._get_optional_kwargs(*args, **kwargs)

        # if no default was supplied, use the parser-level default
        if 'default' not in kwargs:
            dest = kwargs['dest']
            if dest in self._defaults:
                kwargs['default'] = self._defaults[dest]
            elif self.argument_default is not None:
                kwargs['default'] = self.argument_default

        # create the action object, and add it to the parser
        action_class = self._pop_action_class(kwargs)
        if not _callable(action_class):
            raise ValueError('unknown action "%s"' % action_class)
        action = action_class(**kwargs)

        # raise an error if the action type is not callable
        type_func = self._registry_get('type', action.type, action.type)
        if not _callable(type_func):
            raise ValueError('%r is not callable' % type_func)

        return self._add_action(action)


Instruction:
Here you have a function in Python 3, explain what it does
def error(self, message):
        self.print_usage(_sys.stderr)
        self.exit(2, _('%s: error: %s\n') % (self.prog, message))

Response: Print an error message and exit.


Instruction:
Implement a Python 3 function for
giving an object return a boolean indicating whether it is an instance or subclass of Exception.

Response:
def isexception(obj):
    """Given an object, return a boolean indicating whether it is an instance
    or subclass of :py:class:`Exception`.
    """
    if isinstance(obj, Exception):
        return True
    if isclass(obj) and issubclass(obj, Exception):
        return True
    return False


Instruction:
Here you have a function in Python 3, explain what it does
def _reduce_datetimes(row):

    row = list(row)

    for i in range(len(row)):
        if hasattr(row[i], 'isoformat'):
            row[i] = row[i].isoformat()
    return tuple(row)

Response: Receives a row converts datetimes to strings.


Instruction:
Can you tell what is the following Python 3 function doing
def as_dict(self, ordered=False):
        items = zip(self.keys(), self.values())

        return OrderedDict(items) if ordered else dict(items)

Response: Returns the row as a dictionary as ordered.


Instruction:
Can you generate the documentation for the following Python 3 function
def all(self, as_dict=False, as_ordereddict=False):

        # By calling list it calls the __iter__ method
        rows = list(self)

        if as_dict:
            return [r.as_dict() for r in rows]
        elif as_ordereddict:
            return [r.as_dict(ordered=True) for r in rows]

        return rows

Response: Returns a list of all rows for the RecordCollection.


Instruction:
How would you code a function in Python 3 to
return a single record for the RecordCollection ensuring that it is the only record. If default is an instance of Exception then raise it instead of returning it.

Response:
def one(self, default=None, as_dict=False, as_ordereddict=False):
        """Returns a single record for the RecordCollection, ensuring that it
        is the only record, or returns `default`. If `default` is an instance
        or subclass of Exception, then raise it instead of returning it."""

        # Ensure that we don't have more than one row.
        try:
            self[1]
        except IndexError:
            return self.first(default=default, as_dict=as_dict, as_ordereddict=as_ordereddict)
        else:
            raise ValueError('RecordCollection contained more than one row. '
                             'Expects only one row when using '
                             'RecordCollection.one')


Instruction:
Implement a Python 3 function for
getting a connection to this Database.

Response:
def get_connection(self):
        """Get a connection to this Database. Connections are retrieved from a
        pool.
        """
        if not self.open:
            raise exc.ResourceClosedError('Database closed.')

        return Connection(self._engine.connect())


Instruction:
Can you generate a brief explanation for the following Python 3 code
def query(self, query, fetchall=False, **params):
        with self.get_connection() as conn:
            return conn.query(query, fetchall, **params)

Response: Executes a SQL query against the Database. Returns a RecordCollection object.


Instruction:
Can you generate the documentation for the following Python 3 function
def bulk_query(self, query, *multiparams):

        with self.get_connection() as conn:
            conn.bulk_query(query, *multiparams)

Response: Bulk insert or update.


Instruction:
Here you have a function in Python 3, explain what it does
def query_file(self, path, fetchall=False, **params):

        with self.get_connection() as conn:
            return conn.query_file(path, fetchall, **params)

Response: Like Database. query but takes a filename to load a query from.


Instruction:
Write a Python 3 script to
like Database. bulk_query but takes a filename to load a query from.

Response:
def bulk_query_file(self, path, *multiparams):
        """Like Database.bulk_query, but takes a filename to load a query from."""

        with self.get_connection() as conn:
            conn.bulk_query_file(path, *multiparams)


Instruction:
Can you tell what is the following Python 3 function doing
def query(self, query, fetchall=False, **params):

        # Execute the given query.
        cursor = self._conn.execute(text(query), **params) # TODO: PARAMS GO HERE

        # Row-by-row Record generator.
        row_gen = (Record(cursor.keys(), row) for row in cursor)

        # Convert psycopg2 results to RecordCollection.
        results = RecordCollection(row_gen)

        # Fetch all results if desired.
        if fetchall:
            results.all()

        return results

Response: Executes a given SQL query against the connected Database. Returns a RecordCollection which can be iterated over to get result rows as dictionaries.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def bulk_query(self, query, *multiparams):

        self._conn.execute(text(query), *multiparams)

Response: Bulk insert or update.


Instruction:
Can you implement a function in Python 3 that
likes Connection. query but takes a filename to load a query from.

Response:
def query_file(self, path, fetchall=False, **params):
        """Like Connection.query, but takes a filename to load a query from."""

        # If path doesn't exists
        if not os.path.exists(path):
            raise IOError("File '{}' not found!".format(path))

        # If it's a directory
        if os.path.isdir(path):
            raise IOError("'{}' is a directory!".format(path))

        # Read the given .sql file into memory.
        with open(path) as f:
            query = f.read()

        # Defer processing to self.query method.
        return self.query(query=query, fetchall=fetchall, **params)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def bulk_query_file(self, path, *multiparams):

         # If path doesn't exists
        if not os.path.exists(path):
            raise IOError("File '{}'' not found!".format(path))

        # If it's a directory
        if os.path.isdir(path):
            raise IOError("'{}' is a directory!".format(path))

        # Read the given .sql file into memory.
        with open(path) as f:
            query = f.read()

        self._conn.execute(text(query), *multiparams)

Response:
Like Connection. bulk_query but takes a filename to load a query
         from.


Instruction:
Can you create a Python 3 function that
selects the best match in a list or dictionary of choices. Find best matches in a list or dictionary of choices, return a generator of tuples containing the match and its score. If a dictionary is used, also returns the key for each match. Arguments: query: An object representing the thing we want to find. choices: An iterable or dictionary-like object containing choices to be matched against the query. Dictionary arguments of {key: value} pairs will attempt to match the query against each value. processor: Optional function of the form f(a) -> b, where a is the query or individual choice and b is the choice to be used in matching. This can be used to match against, say, the first element of a list: lambda x: x[0] Defaults to fuzzywuzzy.utils.full_process(). scorer: Optional function for scoring matches between the query and an individual processed choice. This should be a function of the form f(query, choice) -> int. By default, fuzz.WRatio() is used and expects both query and choice to be strings. score_cutoff: Optional argument for score threshold. No matches with a score less than this number will be returned. Defaults to 0. Returns: Generator of tuples containing the match and its score. If a list is used for choices, then the result will be 2-tuples. If a dictionary is used, then the result will be 3-tuples containing the key for each match. For example, searching for 'bird' in the dictionary {'bard': 'train', 'dog': 'man'} may return ('train', 22, 'bard'), ('man', 0, 'dog')

Response:
def extractWithoutOrder(query, choices, processor=default_processor, scorer=default_scorer, score_cutoff=0):
    """Select the best match in a list or dictionary of choices.

    Find best matches in a list or dictionary of choices, return a
    generator of tuples containing the match and its score. If a dictionary
    is used, also returns the key for each match.

    Arguments:
        query: An object representing the thing we want to find.
        choices: An iterable or dictionary-like object containing choices
            to be matched against the query. Dictionary arguments of
            {key: value} pairs will attempt to match the query against
            each value.
        processor: Optional function of the form f(a) -> b, where a is the query or
            individual choice and b is the choice to be used in matching.

            This can be used to match against, say, the first element of
            a list:

            lambda x: x[0]

            Defaults to fuzzywuzzy.utils.full_process().
        scorer: Optional function for scoring matches between the query and
            an individual processed choice. This should be a function
            of the form f(query, choice) -> int.

            By default, fuzz.WRatio() is used and expects both query and
            choice to be strings.
        score_cutoff: Optional argument for score threshold. No matches with
            a score less than this number will be returned. Defaults to 0.

    Returns:
        Generator of tuples containing the match and its score.

        If a list is used for choices, then the result will be 2-tuples.
        If a dictionary is used, then the result will be 3-tuples containing
        the key for each match.

        For example, searching for 'bird' in the dictionary

        {'bard': 'train', 'dog': 'man'}

        may return

        ('train', 22, 'bard'), ('man', 0, 'dog')
    """
    # Catch generators without lengths
    def no_process(x):
        return x

    try:
        if choices is None or len(choices) == 0:
            raise StopIteration
    except TypeError:
        pass

    # If the processor was removed by setting it to None
    # perfom a noop as it still needs to be a function
    if processor is None:
        processor = no_process

    # Run the processor on the input query.
    processed_query = processor(query)

    if len(processed_query) == 0:
        logging.warning(u"Applied processor reduces input query to empty string, "
                        "all comparisons will have score 0. "
                        "[Query: \'{0}\']".format(query))

    # Don't run full_process twice
    if scorer in [fuzz.WRatio, fuzz.QRatio,
                  fuzz.token_set_ratio, fuzz.token_sort_ratio,
                  fuzz.partial_token_set_ratio, fuzz.partial_token_sort_ratio,
                  fuzz.UWRatio, fuzz.UQRatio] \
            and processor == utils.full_process:
        processor = no_process

    # Only process the query once instead of for every choice
    if scorer in [fuzz.UWRatio, fuzz.UQRatio]:
        pre_processor = partial(utils.full_process, force_ascii=False)
        scorer = partial(scorer, full_process=False)
    elif scorer in [fuzz.WRatio, fuzz.QRatio,
                    fuzz.token_set_ratio, fuzz.token_sort_ratio,
                    fuzz.partial_token_set_ratio, fuzz.partial_token_sort_ratio]:
        pre_processor = partial(utils.full_process, force_ascii=True)
        scorer = partial(scorer, full_process=False)
    else:
        pre_processor = no_process
    processed_query = pre_processor(processed_query)

    try:
        # See if choices is a dictionary-like object.
        for key, choice in choices.items():
            processed = pre_processor(processor(choice))
            score = scorer(processed_query, processed)
            if score >= score_cutoff:
                yield (choice, score, key)
    except AttributeError:
        # It's a list; just iterate over it.
        for choice in choices:
            processed = pre_processor(processor(choice))
            score = scorer(processed_query, processed)
            if score >= score_cutoff:
                yield (choice, score)


Instruction:
Create a Python 3 function to
select the best match in a list or dictionary of choices and return the key for each match.

Response:
def extract(query, choices, processor=default_processor, scorer=default_scorer, limit=5):
    """Select the best match in a list or dictionary of choices.

    Find best matches in a list or dictionary of choices, return a
    list of tuples containing the match and its score. If a dictionary
    is used, also returns the key for each match.

    Arguments:
        query: An object representing the thing we want to find.
        choices: An iterable or dictionary-like object containing choices
            to be matched against the query. Dictionary arguments of
            {key: value} pairs will attempt to match the query against
            each value.
        processor: Optional function of the form f(a) -> b, where a is the query or
            individual choice and b is the choice to be used in matching.

            This can be used to match against, say, the first element of
            a list:

            lambda x: x[0]

            Defaults to fuzzywuzzy.utils.full_process().
        scorer: Optional function for scoring matches between the query and
            an individual processed choice. This should be a function
            of the form f(query, choice) -> int.
            By default, fuzz.WRatio() is used and expects both query and
            choice to be strings.
        limit: Optional maximum for the number of elements returned. Defaults
            to 5.

    Returns:
        List of tuples containing the match and its score.

        If a list is used for choices, then the result will be 2-tuples.
        If a dictionary is used, then the result will be 3-tuples containing
        the key for each match.

        For example, searching for 'bird' in the dictionary

        {'bard': 'train', 'dog': 'man'}

        may return

        [('train', 22, 'bard'), ('man', 0, 'dog')]
    """
    sl = extractWithoutOrder(query, choices, processor, scorer)
    return heapq.nlargest(limit, sl, key=lambda i: i[1]) if limit is not None else \
        sorted(sl, key=lambda i: i[1], reverse=True)


Instruction:
How would you implement a function in Python 3 that
extracts the best matches to a collection of choices.

Response:
def extractBests(query, choices, processor=default_processor, scorer=default_scorer, score_cutoff=0, limit=5):
    """Get a list of the best matches to a collection of choices.

    Convenience function for getting the choices with best scores.

    Args:
        query: A string to match against
        choices: A list or dictionary of choices, suitable for use with
            extract().
        processor: Optional function for transforming choices before matching.
            See extract().
        scorer: Scoring function for extract().
        score_cutoff: Optional argument for score threshold. No matches with
            a score less than this number will be returned. Defaults to 0.
        limit: Optional maximum for the number of elements returned. Defaults
            to 5.

    Returns: A a list of (match, score) tuples.
    """

    best_list = extractWithoutOrder(query, choices, processor, scorer, score_cutoff)
    return heapq.nlargest(limit, best_list, key=lambda i: i[1]) if limit is not None else \
        sorted(best_list, key=lambda i: i[1], reverse=True)


Instruction:
Write a Python 3 function for
giving a query string and a list of choices returns the single best match and its score.

Response:
def extractOne(query, choices, processor=default_processor, scorer=default_scorer, score_cutoff=0):
    """Find the single best match above a score in a list of choices.

    This is a convenience method which returns the single best choice.
    See extract() for the full arguments list.

    Args:
        query: A string to match against
        choices: A list or dictionary of choices, suitable for use with
            extract().
        processor: Optional function for transforming choices before matching.
            See extract().
        scorer: Scoring function for extract().
        score_cutoff: Optional argument for score threshold. If the best
            match is found, but it is not greater than this number, then
            return None anyway ("not a good enough match").  Defaults to 0.

    Returns:
        A tuple containing a single match and its score, if a match
        was found that was above score_cutoff. Otherwise, returns None.
    """
    best_list = extractWithoutOrder(query, choices, processor, scorer, score_cutoff)
    try:
        return max(best_list, key=lambda i: i[1])
    except ValueError:
        return None


Instruction:
Make a summary of the following Python 3 code
def dedupe(contains_dupes, threshold=70, scorer=fuzz.token_set_ratio):

    extractor = []

    # iterate over items in *contains_dupes*
    for item in contains_dupes:
        # return all duplicate matches found
        matches = extract(item, contains_dupes, limit=None, scorer=scorer)
        # filter matches based on the threshold
        filtered = [x for x in matches if x[1] > threshold]
        # if there is only 1 item in *filtered*, no duplicates were found so append to *extracted*
        if len(filtered) == 1:
            extractor.append(filtered[0][0])

        else:
            # alpha sort
            filtered = sorted(filtered, key=lambda x: x[0])
            # length sort
            filter_sort = sorted(filtered, key=lambda x: len(x[0]), reverse=True)
            # take first item as our 'canonical example'
            extractor.append(filter_sort[0][0])

    # uniquify *extractor* list
    keys = {}
    for e in extractor:
        keys[e] = 1
    extractor = keys.keys()

    # check that extractor differs from contain_dupes (e.g. duplicates were found)
    # if not, then return the original list
    if len(extractor) == len(contains_dupes):
        return contains_dupes
    else:
        return extractor

Response: This function dedupes the duplicate list of the items in contains_dupes and returns a list of the items that are not in the duplicate list.


Instruction:
How would you code a function in Python 3 to
make sure that both objects are either string or unicode instances force them to unicode

Response:
def make_type_consistent(s1, s2):
    """If both objects aren't either both string or unicode instances force them to unicode"""
    if isinstance(s1, str) and isinstance(s2, str):
        return s1, s2

    elif isinstance(s1, unicode) and isinstance(s2, unicode):
        return s1, s2

    else:
        return unicode(s1), unicode(s2)


Instruction:
How would you explain what the following Python 3 function does
def full_process(s, force_ascii=False):

    if force_ascii:
        s = asciidammit(s)
    # Keep only Letters and Numbers (see Unicode docs).
    string_out = StringProcessor.replace_non_letters_non_numbers_with_whitespace(s)
    # Force into lowercase.
    string_out = StringProcessor.to_lower_case(string_out)
    # Remove leading and trailing whitespaces.
    string_out = StringProcessor.strip(string_out)
    return string_out

Response:
Process string by
        and return it as a unicode string.


Instruction:
Can you implement a function in Python 3 that
returns the ratio of the most similar substring as a number between 0 and 100.

Response:
def partial_ratio(s1, s2):
    """"Return the ratio of the most similar substring
    as a number between 0 and 100."""
    s1, s2 = utils.make_type_consistent(s1, s2)

    if len(s1) <= len(s2):
        shorter = s1
        longer = s2
    else:
        shorter = s2
        longer = s1

    m = SequenceMatcher(None, shorter, longer)
    blocks = m.get_matching_blocks()

    # each block represents a sequence of matching characters in a string
    # of the form (idx_1, idx_2, len)
    # the best partial match will block align with at least one of those blocks
    #   e.g. shorter = "abcd", longer = XXXbcdeEEE
    #   block = (1,3,3)
    #   best score === ratio("abcd", "Xbcd")
    scores = []
    for block in blocks:
        long_start = block[1] - block[0] if (block[1] - block[0]) > 0 else 0
        long_end = long_start + len(shorter)
        long_substr = longer[long_start:long_end]

        m2 = SequenceMatcher(None, shorter, long_substr)
        r = m2.ratio()
        if r > .995:
            return 100
        else:
            scores.append(r)

    return utils.intr(100 * max(scores))


Instruction:
Here you have a function in Python 3, explain what it does
def _process_and_sort(s, force_ascii, full_process=True):
    # pull tokens
    ts = utils.full_process(s, force_ascii=force_ascii) if full_process else s
    tokens = ts.split()

    # sort tokens and join
    sorted_string = u" ".join(sorted(tokens))
    return sorted_string.strip()

Response: Return a cleaned string with token sorted.


Instruction:
How would you code a function in Python 3 to
return a measure of the sequences s similarity between 0 and 100 but sorting the token before comparing.

Response:
def token_sort_ratio(s1, s2, force_ascii=True, full_process=True):
    """Return a measure of the sequences' similarity between 0 and 100
    but sorting the token before comparing.
    """
    return _token_sort(s1, s2, partial=False, force_ascii=force_ascii, full_process=full_process)


Instruction:
Given the following Python 3 function, write the documentation
def partial_token_sort_ratio(s1, s2, force_ascii=True, full_process=True):
    return _token_sort(s1, s2, partial=True, force_ascii=force_ascii, full_process=full_process)

Response: Return the ratio of the most similar substring as a number between 0 and 100 but sorting the most similar substring before comparing.


Instruction:
Create a Python 3 function for
returning the set of alphanumeric tokens in the two strings.

Response:
def _token_set(s1, s2, partial=True, force_ascii=True, full_process=True):
    """Find all alphanumeric tokens in each string...
        - treat them as a set
        - construct two strings of the form:
            <sorted_intersection><sorted_remainder>
        - take ratios of those two strings
        - controls for unordered partial matches"""

    if not full_process and s1 == s2:
        return 100

    p1 = utils.full_process(s1, force_ascii=force_ascii) if full_process else s1
    p2 = utils.full_process(s2, force_ascii=force_ascii) if full_process else s2

    if not utils.validate_string(p1):
        return 0
    if not utils.validate_string(p2):
        return 0

    # pull tokens
    tokens1 = set(p1.split())
    tokens2 = set(p2.split())

    intersection = tokens1.intersection(tokens2)
    diff1to2 = tokens1.difference(tokens2)
    diff2to1 = tokens2.difference(tokens1)

    sorted_sect = " ".join(sorted(intersection))
    sorted_1to2 = " ".join(sorted(diff1to2))
    sorted_2to1 = " ".join(sorted(diff2to1))

    combined_1to2 = sorted_sect + " " + sorted_1to2
    combined_2to1 = sorted_sect + " " + sorted_2to1

    # strip
    sorted_sect = sorted_sect.strip()
    combined_1to2 = combined_1to2.strip()
    combined_2to1 = combined_2to1.strip()

    if partial:
        ratio_func = partial_ratio
    else:
        ratio_func = ratio

    pairwise = [
        ratio_func(sorted_sect, combined_1to2),
        ratio_func(sorted_sect, combined_2to1),
        ratio_func(combined_1to2, combined_2to1)
    ]
    return max(pairwise)


Instruction:
Can you tell what is the following Python 3 function doing
def QRatio(s1, s2, force_ascii=True, full_process=True):

    if full_process:
        p1 = utils.full_process(s1, force_ascii=force_ascii)
        p2 = utils.full_process(s2, force_ascii=force_ascii)
    else:
        p1 = s1
        p2 = s2

    if not utils.validate_string(p1):
        return 0
    if not utils.validate_string(p2):
        return 0

    return ratio(p1, p2)

Response: Quick ratio comparison between two strings.


Instruction:
Given the following Python 3 function, write the documentation
def UQRatio(s1, s2, full_process=True):
    return QRatio(s1, s2, force_ascii=False, full_process=full_process)

Response: Unicode quick ratio of two strings


Instruction:
Can you generate the documentation for the following Python 3 function
def WRatio(s1, s2, force_ascii=True, full_process=True):

    if full_process:
        p1 = utils.full_process(s1, force_ascii=force_ascii)
        p2 = utils.full_process(s2, force_ascii=force_ascii)
    else:
        p1 = s1
        p2 = s2

    if not utils.validate_string(p1):
        return 0
    if not utils.validate_string(p2):
        return 0

    # should we look at partials?
    try_partial = True
    unbase_scale = .95
    partial_scale = .90

    base = ratio(p1, p2)
    len_ratio = float(max(len(p1), len(p2))) / min(len(p1), len(p2))

    # if strings are similar length, don't use partials
    if len_ratio < 1.5:
        try_partial = False

    # if one string is much much shorter than the other
    if len_ratio > 8:
        partial_scale = .6

    if try_partial:
        partial = partial_ratio(p1, p2) * partial_scale
        ptsor = partial_token_sort_ratio(p1, p2, full_process=False) \
            * unbase_scale * partial_scale
        ptser = partial_token_set_ratio(p1, p2, full_process=False) \
            * unbase_scale * partial_scale

        return utils.intr(max(base, partial, ptsor, ptser))
    else:
        tsor = token_sort_ratio(p1, p2, full_process=False) * unbase_scale
        tser = token_set_ratio(p1, p2, full_process=False) * unbase_scale

        return utils.intr(max(base, tsor, tser))

Response: Returns a measure of the sequences s similarity between 0 and 100.


Instruction:
Write a Python 3 function for
returning a measure of the sequences s similarity between 0 and 100 using different algorithms.

Response:
def UWRatio(s1, s2, full_process=True):
    """Return a measure of the sequences' similarity between 0 and 100,
    using different algorithms. Same as WRatio but preserving unicode.
    """
    return WRatio(s1, s2, force_ascii=False, full_process=full_process)


Instruction:
How would you implement a function in Python 3 that
prints the result of a timeit statement.

Response:
def print_result_from_timeit(stmt='pass', setup='pass', number=1000000):
    """
    Clean function to know how much time took the execution of one statement
    """
    units = ["s", "ms", "us", "ns"]
    duration = timeit(stmt, setup, number=int(number))
    avg_duration = duration / float(number)
    thousands = int(math.floor(math.log(avg_duration, 1000)))

    print("Total time: %fs. Average run: %.3f%s." % (
        duration, avg_duration * (1000 ** -thousands), units[-thousands]))


Instruction:
Can you create a Python 3 function that
deploys this Model to an Endpoint and optionally return a Predictor.

Response:
def deploy(self, initial_instance_count, instance_type, endpoint_name=None, tags=None):
        """Deploy this ``Model`` to an ``Endpoint`` and optionally return a ``Predictor``.

        Create a SageMaker ``Model`` and ``EndpointConfig``, and deploy an ``Endpoint`` from this ``Model``.
        If ``self.predictor_cls`` is not None, this method returns a the result of invoking
        ``self.predictor_cls`` on the created endpoint name.

        The name of the created model is accessible in the ``name`` field of this ``Model`` after deploy returns

        The name of the created endpoint is accessible in the ``endpoint_name``
        field of this ``Model`` after deploy returns.

        Args:
            instance_type (str): The EC2 instance type to deploy this Model to. For example, 'ml.p2.xlarge'.
            initial_instance_count (int): The initial number of instances to run in the
                ``Endpoint`` created from this ``Model``.
            endpoint_name (str): The name of the endpoint to create (default: None).
                If not specified, a unique endpoint name will be created.
            tags(List[dict[str, str]]): The list of tags to attach to this specific endpoint.

        Returns:
            callable[string, sagemaker.session.Session] or None: Invocation of ``self.predictor_cls`` on
                the created endpoint name, if ``self.predictor_cls`` is not None. Otherwise, return None.
        """
        if not self.sagemaker_session:
            self.sagemaker_session = Session()

        containers = self.pipeline_container_def(instance_type)

        self.name = self.name or name_from_image(containers[0]['Image'])
        self.sagemaker_session.create_model(self.name, self.role, containers, vpc_config=self.vpc_config)

        production_variant = sagemaker.production_variant(self.name, instance_type, initial_instance_count)
        self.endpoint_name = endpoint_name or self.name
        self.sagemaker_session.endpoint_from_production_variants(self.endpoint_name, [production_variant], tags)
        if self.predictor_cls:
            return self.predictor_cls(self.endpoint_name, self.sagemaker_session)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _create_sagemaker_pipeline_model(self, instance_type):
        if not self.sagemaker_session:
            self.sagemaker_session = Session()

        containers = self.pipeline_container_def(instance_type)

        self.name = self.name or name_from_image(containers[0]['Image'])
        self.sagemaker_session.create_model(self.name, self.role, containers, vpc_config=self.vpc_config)

Response: Create a SageMaker Model Entity


Instruction:
Here you have a function in Python 3, explain what it does
def delete_model(self):

        if self.name is None:
            raise ValueError('The SageMaker model must be created before attempting to delete.')

        self.sagemaker_session.delete_model(self.name)

Response: Delete the SageMaker model backing this pipeline model.


Instruction:
Create a Python 3 function to
stream the output of a process to stdout

Response:
def _stream_output(process):
    """Stream the output of a process to stdout

    This function takes an existing process that will be polled for output. Only stdout
    will be polled and sent to sys.stdout.

    Args:
        process(subprocess.Popen): a process that has been started with
            stdout=PIPE and stderr=STDOUT

    Returns (int): process exit code
    """
    exit_code = None

    while exit_code is None:
        stdout = process.stdout.readline().decode("utf-8")
        sys.stdout.write(stdout)
        exit_code = process.poll()

    if exit_code != 0:
        raise RuntimeError("Process exited with code: %s" % exit_code)

    return exit_code


Instruction:
Explain what the following Python 3 code does
def train(self, input_data_config, output_data_config, hyperparameters, job_name):
        self.container_root = self._create_tmp_folder()
        os.mkdir(os.path.join(self.container_root, 'output'))
        # create output/data folder since sagemaker-containers 2.0 expects it
        os.mkdir(os.path.join(self.container_root, 'output', 'data'))
        # A shared directory for all the containers. It is only mounted if the training script is
        # Local.
        shared_dir = os.path.join(self.container_root, 'shared')
        os.mkdir(shared_dir)

        data_dir = self._create_tmp_folder()
        volumes = self._prepare_training_volumes(data_dir, input_data_config, output_data_config,
                                                 hyperparameters)
        # If local, source directory needs to be updated to mounted /opt/ml/code path
        hyperparameters = self._update_local_src_path(hyperparameters, key=sagemaker.estimator.DIR_PARAM_NAME)

        # Create the configuration files for each container that we will create
        # Each container will map the additional local volumes (if any).
        for host in self.hosts:
            _create_config_file_directories(self.container_root, host)
            self.write_config_files(host, hyperparameters, input_data_config)
            shutil.copytree(data_dir, os.path.join(self.container_root, host, 'input', 'data'))

        training_env_vars = {
            REGION_ENV_NAME: self.sagemaker_session.boto_region_name,
            TRAINING_JOB_NAME_ENV_NAME: job_name,
        }
        compose_data = self._generate_compose_file('train', additional_volumes=volumes,
                                                   additional_env_vars=training_env_vars)
        compose_command = self._compose()

        if _ecr_login_if_needed(self.sagemaker_session.boto_session, self.image):
            _pull_image(self.image)

        process = subprocess.Popen(compose_command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)

        try:
            _stream_output(process)
        except RuntimeError as e:
            # _stream_output() doesn't have the command line. We will handle the exception
            # which contains the exit code and append the command line to it.
            msg = "Failed to run: %s, %s" % (compose_command, str(e))
            raise RuntimeError(msg)
        finally:
            artifacts = self.retrieve_artifacts(compose_data, output_data_config, job_name)

            # free up the training data directory as it may contain
            # lots of data downloaded from S3. This doesn't delete any local
            # data that was just mounted to the container.
            dirs_to_delete = [data_dir, shared_dir]
            self._cleanup(dirs_to_delete)

        # Print our Job Complete line to have a similar experience to training on SageMaker where you
        # see this line at the end.
        print('===== Job Complete =====')
        return artifacts

Response: Run a training job locally using Docker - Compose.


Instruction:
Can you tell what is the following Python 3 function doing
def serve(self, model_dir, environment):
        logger.info("serving")

        self.container_root = self._create_tmp_folder()
        logger.info('creating hosting dir in {}'.format(self.container_root))

        volumes = self._prepare_serving_volumes(model_dir)

        # If the user script was passed as a file:// mount it to the container.
        if sagemaker.estimator.DIR_PARAM_NAME.upper() in environment:
            script_dir = environment[sagemaker.estimator.DIR_PARAM_NAME.upper()]
            parsed_uri = urlparse(script_dir)
            if parsed_uri.scheme == 'file':
                volumes.append(_Volume(parsed_uri.path, '/opt/ml/code'))
                # Update path to mount location
                environment = environment.copy()
                environment[sagemaker.estimator.DIR_PARAM_NAME.upper()] = '/opt/ml/code'

        if _ecr_login_if_needed(self.sagemaker_session.boto_session, self.image):
            _pull_image(self.image)

        self._generate_compose_file('serve',
                                    additional_env_vars=environment,
                                    additional_volumes=volumes)
        compose_command = self._compose()
        self.container = _HostingContainer(compose_command)
        self.container.start()

Response: Start a local container using docker - compose.


Instruction:
Can you implement a function in Python 3 that
stops the serving container.

Response:
def stop_serving(self):
        """Stop the serving container.

        The serving container runs in async mode to allow the SDK to do other tasks.
        """
        if self.container:
            self.container.down()
            self.container.join()
            self._cleanup()
        # for serving we can delete everything in the container root.
        _delete_tree(self.container_root)


Instruction:
Can you create a Python 3 function that
gets the model artifacts from all the containers and store them in the artifacts directory.

Response:
def retrieve_artifacts(self, compose_data, output_data_config, job_name):
        """Get the model artifacts from all the container nodes.

        Used after training completes to gather the data from all the individual containers. As the
        official SageMaker Training Service, it will override duplicate files if multiple containers have
        the same file names.

        Args:
            compose_data(dict): Docker-Compose configuration in dictionary format.

        Returns: Local path to the collected model artifacts.

        """
        # We need a directory to store the artfiacts from all the nodes
        # and another one to contained the compressed final artifacts
        artifacts = os.path.join(self.container_root, 'artifacts')
        compressed_artifacts = os.path.join(self.container_root, 'compressed_artifacts')
        os.mkdir(artifacts)

        model_artifacts = os.path.join(artifacts, 'model')
        output_artifacts = os.path.join(artifacts, 'output')

        artifact_dirs = [model_artifacts, output_artifacts, compressed_artifacts]
        for d in artifact_dirs:
            os.mkdir(d)

        # Gather the artifacts from all nodes into artifacts/model and artifacts/output
        for host in self.hosts:
            volumes = compose_data['services'][str(host)]['volumes']
            for volume in volumes:
                host_dir, container_dir = volume.split(':')
                if container_dir == '/opt/ml/model':
                    sagemaker.local.utils.recursive_copy(host_dir, model_artifacts)
                elif container_dir == '/opt/ml/output':
                    sagemaker.local.utils.recursive_copy(host_dir, output_artifacts)

        # Tar Artifacts -> model.tar.gz and output.tar.gz
        model_files = [os.path.join(model_artifacts, name) for name in os.listdir(model_artifacts)]
        output_files = [os.path.join(output_artifacts, name) for name in os.listdir(output_artifacts)]
        sagemaker.utils.create_tar_file(model_files, os.path.join(compressed_artifacts, 'model.tar.gz'))
        sagemaker.utils.create_tar_file(output_files, os.path.join(compressed_artifacts, 'output.tar.gz'))

        if output_data_config['S3OutputPath'] == '':
            output_data = 'file://%s' % compressed_artifacts
        else:
            # Now we just need to move the compressed artifacts to wherever they are required
            output_data = sagemaker.local.utils.move_to_destination(
                compressed_artifacts,
                output_data_config['S3OutputPath'],
                job_name,
                self.sagemaker_session)

        _delete_tree(model_artifacts)
        _delete_tree(output_artifacts)

        return os.path.join(output_data, 'model.tar.gz')


Instruction:
How would you explain what the following Python 3 function does
def write_config_files(self, host, hyperparameters, input_data_config):
        config_path = os.path.join(self.container_root, host, 'input', 'config')

        resource_config = {
            'current_host': host,
            'hosts': self.hosts
        }

        json_input_data_config = {}
        for c in input_data_config:
            channel_name = c['ChannelName']
            json_input_data_config[channel_name] = {
                'TrainingInputMode': 'File'
            }
            if 'ContentType' in c:
                json_input_data_config[channel_name]['ContentType'] = c['ContentType']

        _write_json_file(os.path.join(config_path, 'hyperparameters.json'), hyperparameters)
        _write_json_file(os.path.join(config_path, 'resourceconfig.json'), resource_config)
        _write_json_file(os.path.join(config_path, 'inputdataconfig.json'), json_input_data_config)

Response: This method writes the hyperparameters resources and input data configuration files for the training containers.


Instruction:
How would you implement a function in Python 3 that
writes a config file describing a training/hosting environment. This method generates a docker compose configuration file, it has an entry for each container that will be created (based on self.hosts). it calls :meth:~sagemaker.local_session.SageMakerContainer._create_docker_host to generate the config for each individual container. Args: command (str): either 'train' or 'serve' additional_volumes (list): a list of volumes that will be mapped to the containers additional_env_vars (dict): a dictionary with additional environment variables to be passed on to the containers. Returns: (dict) A dictionary representation of the configuration that was written.

Response:
def _generate_compose_file(self, command, additional_volumes=None, additional_env_vars=None):
        """Writes a config file describing a training/hosting  environment.

        This method generates a docker compose configuration file, it has an entry for each container
        that will be created (based on self.hosts). it calls
        :meth:~sagemaker.local_session.SageMakerContainer._create_docker_host to generate the config
        for each individual container.

        Args:
            command (str): either 'train' or 'serve'
            additional_volumes (list): a list of volumes that will be mapped to the containers
            additional_env_vars (dict): a dictionary with additional environment variables to be
                passed on to the containers.

        Returns: (dict) A dictionary representation of the configuration that was written.

        """
        boto_session = self.sagemaker_session.boto_session
        additional_volumes = additional_volumes or []
        additional_env_vars = additional_env_vars or {}
        environment = []
        optml_dirs = set()

        aws_creds = _aws_credentials(boto_session)
        if aws_creds is not None:
            environment.extend(aws_creds)

        additional_env_var_list = ['{}={}'.format(k, v) for k, v in additional_env_vars.items()]
        environment.extend(additional_env_var_list)

        if os.environ.get(DOCKER_COMPOSE_HTTP_TIMEOUT_ENV) is None:
            os.environ[DOCKER_COMPOSE_HTTP_TIMEOUT_ENV] = DOCKER_COMPOSE_HTTP_TIMEOUT

        if command == 'train':
            optml_dirs = {'output', 'output/data', 'input'}

        services = {
            h: self._create_docker_host(h, environment, optml_dirs,
                                        command, additional_volumes) for h in self.hosts
        }

        content = {
            # Use version 2.3 as a minimum so that we can specify the runtime
            'version': '2.3',
            'services': services,
            'networks': {
                'sagemaker-local': {'name': 'sagemaker-local'}
            }
        }

        docker_compose_path = os.path.join(self.container_root, DOCKER_COMPOSE_FILENAME)
        yaml_content = yaml.dump(content, default_flow_style=False)
        logger.info('docker compose file: \n{}'.format(yaml_content))
        with open(docker_compose_path, 'w') as f:
            f.write(yaml_content)

        return content


Instruction:
Write a Python 3 script to
build a list of volumes that need to be training for the container.

Response:
def _build_optml_volumes(self, host, subdirs):
        """Generate a list of :class:`~sagemaker.local_session.Volume` required for the container to start.

        It takes a folder with the necessary files for training and creates a list of opt volumes that
        the Container needs to start.

        Args:
            host (str): container for which the volumes will be generated.
            subdirs (list): list of subdirectories that will be mapped. For example: ['input', 'output', 'model']

        Returns: (list) List of :class:`~sagemaker.local_session.Volume`
        """
        volumes = []

        for subdir in subdirs:
            host_dir = os.path.join(self.container_root, host, subdir)
            container_dir = '/opt/ml/{}'.format(subdir)
            volume = _Volume(host_dir, container_dir)
            volumes.append(volume)

        return volumes


Instruction:
Explain what the following Python 3 code does
def model_fn(model_dir):
    symbol = mx.sym.load('%s/model.json' % model_dir)
    outputs = mx.symbol.softmax(data=symbol, name='softmax_label')
    inputs = mx.sym.var('data')
    param_dict = gluon.ParameterDict('model_')
    net = gluon.SymbolBlock(outputs, inputs, param_dict)
    net.load_params('%s/model.params' % model_dir, ctx=mx.cpu())
    return net

Response: Loads the gluon model. Called once when hosting service starts.


Instruction:
Here you have a function in Python 3, explain what it does
def transform_fn(net, data, input_content_type, output_content_type):
    # we can use content types to vary input/output handling, but
    # here we just assume json for both
    parsed = json.loads(data)
    nda = mx.nd.array(parsed)
    output = net(nda)
    prediction = mx.nd.argmax(output, axis=1)
    response_body = json.dumps(prediction.asnumpy().tolist())
    return response_body, output_content_type

Response: Transform a request using the Gluon model. Called once per request.


Instruction:
How would you explain what the following Python 3 function does
def _prepare_for_training(self, records, mini_batch_size=None, job_name=None):
        num_records = None
        if isinstance(records, list):
            for record in records:
                if record.channel == 'train':
                    num_records = record.num_records
                    break
            if num_records is None:
                raise ValueError('Must provide train channel.')
        else:
            num_records = records.num_records

        # mini_batch_size is a required parameter
        default_mini_batch_size = min(self.DEFAULT_MINI_BATCH_SIZE,
                                      max(1, int(num_records / self.train_instance_count)))
        use_mini_batch_size = mini_batch_size or default_mini_batch_size

        super(PCA, self)._prepare_for_training(records=records, mini_batch_size=use_mini_batch_size, job_name=job_name)

Response: Set hyperparameters needed for training.


Instruction:
Can you generate the documentation for the following Python 3 function
def create_image_uri(region, framework, instance_type, framework_version, py_version=None,
                     account='520713654638', accelerator_type=None, optimized_families=None):
    """Return the ECR URI of an image.

    Args:
        region (str): AWS region where the image is uploaded.
        framework (str): framework used by the image.
        instance_type (str): SageMaker instance type. Used to determine device type (cpu/gpu/family-specific optimized).
        framework_version (str): The version of the framework.
        py_version (str): Optional. Python version. If specified, should be one of 'py2' or 'py3'.
            If not specified, image uri will not include a python component.
        account (str): AWS account that contains the image. (default: '520713654638')
        accelerator_type (str): SageMaker Elastic Inference accelerator type.
        optimized_families (str): Instance families for which there exist specific optimized images.

    Returns:
        str: The appropriate image URI based on the given parameters.
    """
    optimized_families = optimized_families or []

    if py_version and py_version not in VALID_PY_VERSIONS:
        raise ValueError('invalid py_version argument: {}'.format(py_version))

    # Handle Account Number for Gov Cloud
    account = VALID_ACCOUNTS_BY_REGION.get(region, account)

    # Handle Local Mode
    if instance_type.startswith('local'):
        device_type = 'cpu' if instance_type == 'local' else 'gpu'
    elif not instance_type.startswith('ml.'):
        raise ValueError('{} is not a valid SageMaker instance type. See: '
                         'https://aws.amazon.com/sagemaker/pricing/instance-types/'.format(instance_type))
    else:
        family = instance_type.split('.')[1]

        # For some frameworks, we have optimized images for specific families, e.g c5 or p3. In those cases,
        # we use the family name in the image tag. In other cases, we use 'cpu' or 'gpu'.
        if family in optimized_families:
            device_type = family
        elif family[0] in ['g', 'p']:
            device_type = 'gpu'
        else:
            device_type = 'cpu'

    if py_version:
        tag = "{}-{}-{}".format(framework_version, device_type, py_version)
    else:
        tag = "{}-{}".format(framework_version, device_type)

    if _accelerator_type_valid_for_framework(framework=framework, accelerator_type=accelerator_type,
                                             optimized_families=optimized_families):
        framework += '-eia'

    return "{}/sagemaker-{}:{}" \
        .format(get_ecr_image_uri_prefix(account, region), framework, tag)

Response: Create an image URI based on the given parameters.


Instruction:
Explain what the following Python 3 code does
def validate_source_dir(script, directory):
    if directory:
        if not os.path.isfile(os.path.join(directory, script)):
            raise ValueError('No file named "{}" was found in directory "{}".'.format(script, directory))

    return True

Response: Validate that the source directory exists and contains the user script.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def tar_and_upload_dir(session, bucket, s3_key_prefix, script,
                       directory=None, dependencies=None, kms_key=None):
    """Package source files and upload a compress tar file to S3. The S3 location will be
    ``s3://<bucket>/s3_key_prefix/sourcedir.tar.gz``.

    If directory is an S3 URI, an UploadedCode object will be returned, but nothing will be
    uploaded to S3 (this allow reuse of code already in S3).

    If directory is None, the script will be added to the archive at ``./<basename of script>``.

    If directory is not None, the (recursive) contents of the directory will be added to
    the archive. directory is treated as the base path of the archive, and the script name is
    assumed to be a filename or relative path inside the directory.

    Args:
        session (boto3.Session): Boto session used to access S3.
        bucket (str): S3 bucket to which the compressed file is uploaded.
        s3_key_prefix (str): Prefix for the S3 key.
        script (str): Script filename or path.
        directory (str): Optional. Directory containing the source file. If it starts with "s3://",
            no action is taken.
        dependencies (List[str]): Optional. A list of paths to directories (absolute or relative)
                                containing additional libraries that will be copied into
                                /opt/ml/lib
        kms_key (str):  Optional. KMS key ID used to upload objects to the bucket (default: None).

    Returns:
        sagemaker.fw_utils.UserCode: An object with the S3 bucket and key (S3 prefix) and
            script name.
    """
    if directory and directory.lower().startswith('s3://'):
        return UploadedCode(s3_prefix=directory, script_name=os.path.basename(script))

    script_name = script if directory else os.path.basename(script)
    dependencies = dependencies or []
    key = '%s/sourcedir.tar.gz' % s3_key_prefix
    tmp = tempfile.mkdtemp()

    try:
        source_files = _list_files_to_compress(script, directory) + dependencies
        tar_file = sagemaker.utils.create_tar_file(source_files,
                                                   os.path.join(tmp, _TAR_SOURCE_FILENAME))

        if kms_key:
            extra_args = {'ServerSideEncryption': 'aws:kms', 'SSEKMSKeyId': kms_key}
        else:
            extra_args = None

        session.resource('s3').Object(bucket, key).upload_file(tar_file, ExtraArgs=extra_args)
    finally:
        shutil.rmtree(tmp)

    return UploadedCode(s3_prefix='s3://%s/%s' % (bucket, key), script_name=script_name)

Response: Package source files and upload a tar file to S3.


Instruction:
Make a summary of the following Python 3 code
def framework_name_from_image(image_name):
    sagemaker_pattern = re.compile(ECR_URI_PATTERN)
    sagemaker_match = sagemaker_pattern.match(image_name)
    if sagemaker_match is None:
        return None, None, None, None
    else:
        # extract framework, python version and image tag
        # We must support both the legacy and current image name format.
        name_pattern = re.compile(
            r'^sagemaker(?:-rl)?-(tensorflow|mxnet|chainer|pytorch|scikit-learn)(?:-)?(scriptmode)?:(.*)-(.*?)-(py2|py3)$') # noqa
        legacy_name_pattern = re.compile(
            r'^sagemaker-(tensorflow|mxnet)-(py2|py3)-(cpu|gpu):(.*)$')

        name_match = name_pattern.match(sagemaker_match.group(9))
        legacy_match = legacy_name_pattern.match(sagemaker_match.group(9))

        if name_match is not None:
            fw, scriptmode, ver, device, py = name_match.group(1), name_match.group(2), name_match.group(3),\
                name_match.group(4), name_match.group(5)
            return fw, py, '{}-{}-{}'.format(ver, device, py), scriptmode
        elif legacy_match is not None:
            return legacy_match.group(1), legacy_match.group(2), legacy_match.group(4), None
        else:
            return None, None, None, None

Response: Extract the framework and Python version from the image name.


Instruction:
Implement a Python 3 function for
extracting the framework version from the image tag.

Response:
def framework_version_from_tag(image_tag):
    """Extract the framework version from the image tag.

    Args:
        image_tag (str): Image tag, which should take the form '<framework_version>-<device>-<py_version>'

    Returns:
        str: The framework version.
    """
    tag_pattern = re.compile('^(.*)-(cpu|gpu)-(py2|py3)$')
    tag_match = tag_pattern.match(image_tag)
    return None if tag_match is None else tag_match.group(1)


Instruction:
How would you code a function in Python 3 to
parse an s3 url and return a tuple containing the bucket name and prefix

Response:
def parse_s3_url(url):
    """Returns an (s3 bucket, key name/prefix) tuple from a url with an s3 scheme

    Args:
        url (str):

    Returns:
        tuple: A tuple containing:
            str: S3 bucket name
            str: S3 key
    """
    parsed_url = urlparse(url)
    if parsed_url.scheme != "s3":
        raise ValueError("Expecting 's3' scheme, got: {} in {}".format(parsed_url.scheme, url))
    return parsed_url.netloc, parsed_url.path.lstrip('/')


Instruction:
Here you have a function in Python 3, explain what it does
def model_code_key_prefix(code_location_key_prefix, model_name, image):
    training_job_name = sagemaker.utils.name_from_image(image)
    return '/'.join(filter(None, [code_location_key_prefix, model_name or training_job_name]))

Response: Returns the s3 key prefix for uploading code during model deployment


Instruction:
Can you tell what is the following Python 3 function doing
def create_training_job(self, TrainingJobName, AlgorithmSpecification, OutputDataConfig,
                            ResourceConfig, InputDataConfig=None, **kwargs):
        """
        Create a training job in Local Mode
        Args:
            TrainingJobName (str): local training job name.
            AlgorithmSpecification (dict): Identifies the training algorithm to use.
            InputDataConfig (dict): Describes the training dataset and the location where it is stored.
            OutputDataConfig (dict): Identifies the location where you want to save the results of model training.
            ResourceConfig (dict): Identifies the resources to use for local model traininig.
            HyperParameters (dict) [optional]: Specifies these algorithm-specific parameters to influence the quality of
                the final model.
        """
        InputDataConfig = InputDataConfig or {}
        container = _SageMakerContainer(ResourceConfig['InstanceType'], ResourceConfig['InstanceCount'],
                                        AlgorithmSpecification['TrainingImage'], self.sagemaker_session)
        training_job = _LocalTrainingJob(container)
        hyperparameters = kwargs['HyperParameters'] if 'HyperParameters' in kwargs else {}
        training_job.start(InputDataConfig, OutputDataConfig, hyperparameters, TrainingJobName)

        LocalSagemakerClient._training_jobs[TrainingJobName] = training_job

Response: Creates a training job in Local Mode.