Instruction:
How would you implement a function in Python 3 that
fits KimCNNClassifier according to X y

Response:
def fit(self, X, y):
        """Fit KimCNNClassifier according to X, y

        Parameters
        ----------
        X : list of string
            each item is a raw text
        y : list of string
            each item is a label
        """
        ####################
        # Data Loader
        ####################
        word_vector_transformer = WordVectorTransformer(padding='max')
        X = word_vector_transformer.fit_transform(X)
        X = LongTensor(X)
        self.word_vector_transformer = word_vector_transformer

        y_transformer = LabelEncoder()
        y = y_transformer.fit_transform(y)
        y = torch.from_numpy(y)
        self.y_transformer = y_transformer

        dataset = CategorizedDataset(X, y)
        dataloader = DataLoader(dataset,
                                batch_size=self.batch_size,
                                shuffle=True,
                                num_workers=4)

        ####################
        # Model
        ####################
        KERNEL_SIZES = self.kernel_sizes
        NUM_KERNEL = self.num_kernel
        EMBEDDING_DIM = self.embedding_dim

        model = TextCNN(
            vocab_size=word_vector_transformer.get_vocab_size(),
            embedding_dim=EMBEDDING_DIM,
            output_size=len(self.y_transformer.classes_),
            kernel_sizes=KERNEL_SIZES,
            num_kernel=NUM_KERNEL)
        if USE_CUDA:
            model = model.cuda()

        ####################
        # Train
        ####################
        EPOCH = self.epoch
        LR = self.lr

        loss_function = nn.CrossEntropyLoss()
        optimizer = optim.Adam(model.parameters(), lr=LR)

        for epoch in range(EPOCH):
            losses = []
            for i, data in enumerate(dataloader):
                X, y = data
                X, y = Variable(X), Variable(y)

                optimizer.zero_grad()
                model.train()
                output = model(X)

                loss = loss_function(output, y)
                losses.append(loss.data.tolist()[0])
                loss.backward()

                optimizer.step()

                if i % 100 == 0:
                    print("[%d/%d] mean_loss : %0.2f" % (
                        epoch, EPOCH, np.mean(losses)))
                    losses = []
        self.model = model


Instruction:
Here you have a function in Python 3, explain what it does
def config_sources(app, environment, cluster, configs_dirs, app_dir,
                   local=False, build=False):
    """Return the config files for an environment & cluster specific app."""
    sources = [
        # Machine-specific
        (configs_dirs, 'hostname'),
        (configs_dirs, 'hostname-local'),
        (configs_dirs, 'hostname-build'),
        # Global
        (configs_dirs, 'common'),
        # Environment + Cluster
        (configs_dirs, 'common-%s' % environment),
        (configs_dirs, 'common-%s-%s' % (environment, cluster)),
        (configs_dirs, 'common-local'),
        (configs_dirs, 'common-build'),
        # Machine-specific overrides
        (configs_dirs, 'common-overrides'),
        # Application-specific
        ([app_dir], '%s-default' % app),
        ([app_dir], '%s-%s' % (app, environment)),
        ([app_dir], '%s-%s-%s' % (app, environment, cluster)),
        (configs_dirs, app),
        (configs_dirs, '%s-%s' % (app, environment)),
        (configs_dirs, '%s-%s-%s' % (app, environment, cluster)),
        ([app_dir], '%s-local' % app),
        ([app_dir], '%s-build' % app),
        (configs_dirs, '%s-local' % app),
        (configs_dirs, '%s-build' % app),
        # Machine-specific application override
        (configs_dirs, '%s-overrides' % app),
    ]

    # Filter out build sources if not requested
    if not build:
        sources = [source for source in sources
                   if not source[1].endswith('-build')]
        # Filter out local sources if not build and not local
        if not local:
            sources = [source for source in sources
                       if not source[1].endswith('-local')]

    return available_sources(sources)

Response: Return the config files for an application environment & cluster specific app.


Instruction:
Create a Python 3 function for
yielding the sources that are present in the source directory.

Response:
def available_sources(sources):
    """Yield the sources that are present."""
    for dirs, name in sources:
        for directory in dirs:
            fn = os.path.join(directory, name) + '.py'
            if os.path.isfile(fn):
                yield fn


Instruction:
Implement a function in Python 3 to
merge the configuration sources and return the resulting DotDict.

Response:
def smush_config(sources, initial=None):
    """Merge the configuration sources and return the resulting DotDict."""
    if initial is None:
        initial = {}
    config = DotDict(initial)

    for fn in sources:
        log.debug('Merging %s', fn)
        mod = get_config_module(fn)
        config = mod.update(config)
        log.debug('Current config:\n%s', json.dumps(config, indent=4,
                                                    cls=LenientJSONEncoder))
    return config


Instruction:
How would you code a function in Python 3 to
merge dictionary d2 into d1 overriding entries in d1 with values from d2.

Response:
def merge_dicts(d1, d2, _path=None):
    """
    Merge dictionary d2 into d1, overriding entries in d1 with values from d2.

    d1 is mutated.

    _path is for internal, recursive use.
    """
    if _path is None:
        _path = ()
    if isinstance(d1, dict) and isinstance(d2, dict):
        for k, v in d2.items():
            if isinstance(v, MissingValue) and v.name is None:
                v.name = '.'.join(_path + (k,))

            if isinstance(v, DeletedValue):
                d1.pop(k, None)
            elif k not in d1:
                if isinstance(v, dict):
                    d1[k] = merge_dicts({}, v, _path + (k,))
                else:
                    d1[k] = v
            else:
                if isinstance(d1[k], dict) and isinstance(v, dict):
                    d1[k] = merge_dicts(d1[k], v, _path + (k,))
                elif isinstance(d1[k], list) and isinstance(v, list):
                    # Lists are only supported as leaves
                    d1[k] += v
                elif isinstance(d1[k], MissingValue):
                    d1[k] = v
                elif d1[k] is None:
                    d1[k] = v
                elif type(d1[k]) == type(v):
                    d1[k] = v
                else:
                    raise TypeError('Refusing to replace a %s with a %s'
                                    % (type(d1[k]), type(v)))
    else:
        raise TypeError('Cannot merge a %s with a %s' % (type(d1), type(d2)))

    return d1


Instruction:
Explain what the following Python 3 code does
def filter_dict(unfiltered, filter_keys):
    filtered = DotDict()
    for k in filter_keys:
        filtered[k] = unfiltered[k]
    return filtered

Response: Return a subset of a dictionary using the specified keys.


Instruction:
Explain what the following Python 3 code does
def _convert_item(self, obj):
        if isinstance(obj, dict) and not isinstance(obj, DotDict):
            obj = DotDict(obj)
        elif isinstance(obj, list):
            # must mutate and not just reassign, otherwise it will
            # just use original object mutable/immutable
            for i, item in enumerate(obj):
                if isinstance(item, dict) and not isinstance(item, DotDict):
                    obj[i] = DotDict(item)
        return obj

Response:
Convert obj into a DotDict or list of DotDict.

        Directly nested lists aren t supported.


Instruction:
Implement a Python 3 function for
returning a config subset using the filter defined in the deploy config.

Response:
def filter_config(config, deploy_config):
    """Return a config subset using the filter defined in the deploy config."""
    if not os.path.isfile(deploy_config):
        return DotDict()
    config_module = get_config_module(deploy_config)
    return config_module.filter(config)


Instruction:
Make a summary of the following Python 3 code
def seeded_auth_token(client, service, seed):
    hash_func = hashlib.md5()
    token = ','.join((client, service, seed)).encode('utf-8')
    hash_func.update(token)
    return hash_func.hexdigest()

Response: Return an auth token based on the client + service + seed tuple.


Instruction:
How would you explain what the following Python 3 function does
def write_config(config, app_dir, filename='configuration.json'):
    path = os.path.join(app_dir, filename)
    with open(path, 'w') as f:
        json.dump(
            config, f, indent=4, cls=DetectMissingEncoder,
            separators=(',', ': '))

Response: Write the configuration to the applicaiton directory.


Instruction:
Create a Python 3 function to
import the config file to yoconfigurator. configs.

Response:
def get_config_module(config_pathname):
    """Imports the config file to yoconfigurator.configs.<config_basename>."""
    configs_mod = 'yoconfigurator.configs'
    if configs_mod not in sys.modules:
        sys.modules[configs_mod] = types.ModuleType(configs_mod)
    module_name = os.path.basename(config_pathname).rsplit('.', 1)[0]
    module_name = configs_mod + '.' + module_name
    return _load_module(module_name, config_pathname)


Instruction:
Here you have a function in Python 3, explain what it does
def validate_date(date_text):
    try:
        if int(date_text) < 0:
            return True
    except ValueError:
        pass

    try:
        datetime.strptime(date_text, '%Y-%m-%d')
        return True
    except ValueError:
        pass

    raise ValueError('Dates must be negative integers or YYYY-MM-DD in the past.')

Response: Return True if valid raise ValueError if not


Instruction:
Given the following Python 3 function, write the documentation
def get_download_total(rows):
    headers = rows.pop(0)
    index = headers.index('download_count')
    total_downloads = sum(int(row[index]) for row in rows)

    rows.insert(0, headers)
    return total_downloads, index

Response: Return the total downloads and the downloads column


Instruction:
Write a Python 3 script for
adding a final row to rows showing the total downloads

Response:
def add_download_total(rows):
    """Add a final row to rows showing the total downloads"""
    total_row = [""] * len(rows[0])
    total_row[0] = "Total"
    total_downloads, downloads_column = get_download_total(rows)
    total_row[downloads_column] = str(total_downloads)
    rows.append(total_row)

    return rows


Instruction:
Write a Python 3 script for
getting pypinfo for a single object.

Response:
def pypinfo(
    ctx,
    project,
    fields,
    auth,
    run,
    json,
    indent,
    timeout,
    limit,
    days,
    start_date,
    end_date,
    where,
    order,
    all_installers,
    percent,
    markdown,
):
    """Valid fields are:\n
    project | version | file | pyversion | percent3 | percent2 | impl | impl-version |\n
    openssl | date | month | year | country | installer | installer-version |\n
    setuptools-version | system | system-release | distro | distro-version | cpu
    """
    if auth:
        set_credentials(auth)
        click.echo('Credentials location set to "{}".'.format(get_credentials()))
        return

    if project is None and not fields:
        click.echo(ctx.get_help())
        return

    parsed_fields = []
    for field in fields:
        parsed = FIELD_MAP.get(field)
        if parsed is None:
            raise ValueError('"{}" is an unsupported field.'.format(field))
        parsed_fields.append(parsed)

    order_name = order
    order = FIELD_MAP.get(order)
    if order:
        order_name = order.name
        parsed_fields.insert(0, order)

    built_query = build_query(
        project,
        parsed_fields,
        limit=limit,
        days=days,
        start_date=start_date,
        end_date=end_date,
        where=where,
        order=order_name,
        pip=not all_installers,
    )

    if run:
        client = create_client(get_credentials())
        query_job = client.query(built_query, job_config=create_config())
        query_rows = query_job.result(timeout=timeout // 1000)

        # Cached
        from_cache = not not query_job.cache_hit

        # Processed
        bytes_processed = query_job.total_bytes_processed or 0
        processed_amount, processed_unit = convert_units(bytes_processed)

        # Billed
        bytes_billed = query_job.total_bytes_billed or 0
        billed_amount, billed_unit = convert_units(bytes_billed)

        # Cost
        billing_tier = query_job.billing_tier or 1
        estimated_cost = Decimal(TIER_COST * billing_tier) / TB * Decimal(bytes_billed)
        estimated_cost = str(estimated_cost.quantize(TO_CENTS, rounding=ROUND_UP))

        rows = parse_query_result(query_job, query_rows)

        if percent:
            rows = add_percentages(rows, include_sign=not json)

        # Only for tables, and if more than the header row + a single data row
        if len(rows) > 2 and not json:
            rows = add_download_total(rows)

        if not json:
            click.echo('Served from cache: {}'.format(from_cache))
            click.echo('Data processed: {:.2f} {}'.format(processed_amount, processed_unit))
            click.echo('Data billed: {:.2f} {}'.format(billed_amount, billed_unit))
            click.echo('Estimated cost: ${}'.format(estimated_cost))

            click.echo()
            click.echo(tabulate(rows, markdown))
        else:
            query_info = {
                'cached': from_cache,
                'bytes_processed': bytes_processed,
                'bytes_billed': bytes_billed,
                'estimated_cost': estimated_cost,
            }
            click.echo(format_json(rows, query_info, indent))
    else:
        click.echo(built_query)


Instruction:
Write a Python 3 function for
finding and patch an entry in the soup.

Response:
def find_and_patch_entry(soup, entry):
    """
    Modify soup so Dash.app can generate TOCs on the fly.
    """
    link = soup.find("a", {"class": "headerlink"}, href="#" + entry.anchor)
    tag = soup.new_tag("a")
    tag["name"] = APPLE_REF_TEMPLATE.format(entry.type, entry.name)
    if link:
        link.parent.insert(0, tag)
        return True
    elif entry.anchor.startswith("module-"):
        soup.h1.parent.insert(0, tag)
        return True
    else:
        return False


Instruction:
Make a summary of the following Python 3 code
def inv_entry_to_path(data):
    path_tuple = data[2].split("#")
    if len(path_tuple) > 1:
        path_str = "#".join((path_tuple[0], path_tuple[-1]))
    else:
        path_str = data[2]
    return path_str

Response: Convert an intersphinx inventory entry to a path string.


Instruction:
Explain what the following Python 3 code does
def main(
    source,
    force,
    name,
    quiet,
    verbose,
    destination,
    add_to_dash,
    add_to_global,
    icon,
    index_page,
    enable_js,
    online_redirect_url,
    parser,
):
    """
    Convert docs from SOURCE to Dash.app's docset format.
    """
    try:
        logging.config.dictConfig(
            create_log_config(verbose=verbose, quiet=quiet)
        )
    except ValueError as e:
        click.secho(e.args[0], fg="red")
        raise SystemExit(1)

    if icon:
        icon_data = icon.read()
        if not icon_data.startswith(PNG_HEADER):
            log.error(
                '"{}" is not a valid PNG image.'.format(
                    click.format_filename(icon.name)
                )
            )
            raise SystemExit(1)
    else:
        icon_data = None

    source, dest, name = setup_paths(
        source,
        destination,
        name=name,
        add_to_global=add_to_global,
        force=force,
    )
    if parser is None:
        parser = parsers.get_doctype(source)
        if parser is None:
            log.error(
                '"{}" does not contain a known documentation format.'.format(
                    click.format_filename(source)
                )
            )
            raise SystemExit(errno.EINVAL)
    docset = prepare_docset(
        source, dest, name, index_page, enable_js, online_redirect_url
    )
    doc_parser = parser(doc_path=docset.docs)
    log.info(
        (
            "Converting "
            + click.style("{parser_name}", bold=True)
            + ' docs from "{src}" to "{dst}".'
        ).format(
            parser_name=parser.name,
            src=click.format_filename(source, shorten=True),
            dst=click.format_filename(dest),
        )
    )

    with docset.db_conn:
        log.info("Parsing documentation...")
        toc = patch_anchors(doc_parser, show_progressbar=not quiet)
        for entry in doc_parser.parse():
            docset.db_conn.execute(
                "INSERT INTO searchIndex VALUES (NULL, ?, ?, ?)",
                entry.as_tuple(),
            )
            toc.send(entry)
        count = docset.db_conn.execute(
            "SELECT COUNT(1) FROM searchIndex"
        ).fetchone()[0]
        log.info(
            (
                "Added "
                + click.style("{count:,}", fg="green" if count > 0 else "red")
                + " index entries."
            ).format(count=count)
        )
        toc.close()

    if icon_data:
        add_icon(icon_data, dest)

    if add_to_dash or add_to_global:
        log.info("Adding to Dash.app...")
        os.system('open -a dash "{}"'.format(dest))

Response: Convert a source file to Dash. app s docset format.


Instruction:
Write a Python 3 function that can
create a log config dictionary.

Response:
def create_log_config(verbose, quiet):
    """
    We use logging's levels as an easy-to-use verbosity controller.
    """
    if verbose and quiet:
        raise ValueError(
            "Supplying both --quiet and --verbose makes no sense."
        )
    elif verbose:
        level = logging.DEBUG
    elif quiet:
        level = logging.ERROR
    else:
        level = logging.INFO

    logger_cfg = {"handlers": ["click_handler"], "level": level}

    return {
        "version": 1,
        "formatters": {"click_formatter": {"format": "%(message)s"}},
        "handlers": {
            "click_handler": {
                "level": level,
                "class": "doc2dash.__main__.ClickEchoHandler",
                "formatter": "click_formatter",
            }
        },
        "loggers": {"doc2dash": logger_cfg, "__main__": logger_cfg},
    }


Instruction:
How would you implement a function in Python 3 that
determines source and destination using the options.

Response:
def setup_paths(source, destination, name, add_to_global, force):
    """
    Determine source and destination using the options.
    """
    if source[-1] == "/":
        source = source[:-1]
    if not name:
        name = os.path.split(source)[-1]
    elif name.endswith(".docset"):
        name = name.replace(".docset", "")
    if add_to_global:
        destination = DEFAULT_DOCSET_PATH
    dest = os.path.join(destination or "", name + ".docset")
    dst_exists = os.path.lexists(dest)
    if dst_exists and force:
        shutil.rmtree(dest)
    elif dst_exists:
        log.error(
            'Destination path "{}" already exists.'.format(
                click.format_filename(dest)
            )
        )
        raise SystemExit(errno.EEXIST)
    return source, dest, name


Instruction:
Can you generate the documentation for the following Python 3 function
def prepare_docset(
    source, dest, name, index_page, enable_js, online_redirect_url
):
    """
    Create boilerplate files & directories and copy vanilla docs inside.

    Return a tuple of path to resources and connection to sqlite db.
    """
    resources = os.path.join(dest, "Contents", "Resources")
    docs = os.path.join(resources, "Documents")
    os.makedirs(resources)

    db_conn = sqlite3.connect(os.path.join(resources, "docSet.dsidx"))
    db_conn.row_factory = sqlite3.Row
    db_conn.execute(
        "CREATE TABLE searchIndex(id INTEGER PRIMARY KEY, name TEXT, "
        "type TEXT, path TEXT)"
    )
    db_conn.commit()

    plist_path = os.path.join(dest, "Contents", "Info.plist")
    plist_cfg = {
        "CFBundleIdentifier": name,
        "CFBundleName": name,
        "DocSetPlatformFamily": name.lower(),
        "DashDocSetFamily": "python",
        "isDashDocset": True,
        "isJavaScriptEnabled": enable_js,
    }
    if index_page is not None:
        plist_cfg["dashIndexFilePath"] = index_page
    if online_redirect_url is not None:
        plist_cfg["DashDocSetFallbackURL"] = online_redirect_url

    write_plist(plist_cfg, plist_path)

    shutil.copytree(source, docs)

    return DocSet(path=dest, docs=docs, plist=plist_path, db_conn=db_conn)

Response: Prepare a new DocSet from a source file to a destination file.


Instruction:
Given the following Python 3 function, write the documentation
def add_icon(icon_data, dest):
    with open(os.path.join(dest, "icon.png"), "wb") as f:
        f.write(icon_data)

Response: Add icon to docset


Instruction:
Given the following Python 3 function, write the documentation
def patch_anchors(parser, show_progressbar):
    files = defaultdict(list)
    try:
        while True:
            pentry = (yield)
            try:
                fname, anchor = pentry.path.split("#")
                files[fname].append(
                    TOCEntry(name=pentry.name, type=pentry.type, anchor=anchor)
                )
            except ValueError:
                # pydoctor has no anchors for e.g. classes
                pass
    except GeneratorExit:
        pass

    def patch_files(files):
        for fname, entries in files:
            full_path = os.path.join(parser.doc_path, fname)
            with codecs.open(full_path, mode="r", encoding="utf-8") as fp:
                soup = BeautifulSoup(fp, "html.parser")
                for entry in entries:
                    if not parser.find_and_patch_entry(soup, entry):
                        log.debug(
                            "Can't find anchor {} in {}.".format(
                                entry.anchor, click.format_filename(fname)
                            )
                        )
            with open(full_path, mode="wb") as fp:
                fp.write(soup.encode("utf-8"))

    if show_progressbar is True:
        with click.progressbar(
            files.items(),
            width=0,
            length=len(files),
            label="Adding table of contents meta data...",
        ) as pbar:
            patch_files(pbar)
    else:
        patch_files(files.items())

Response: Consume parse entries and patch the anchor names of the TOCs.


Instruction:
Here you have a function in Python 3, explain what it does
def has_file_with(path, filename, content):
    try:
        with open(os.path.join(path, filename), "rb") as f:
            return content in f.read()
    except IOError as e:
        if e.errno == errno.ENOENT:
            return False
        else:
            raise

Response: Check whether filename in path contains the string content.


Instruction:
Explain what the following Python 3 code does
def run_cell(self, cell):
        globals = self.ipy_shell.user_global_ns
        locals = self.ipy_shell.user_ns
        globals.update({
            "__ipy_scope__": None,
        })
        try:
            with redirect_stdout(self.stdout):
                self.run(cell, globals, locals)
        except:
            self.code_error = True
            if self.options.debug:
                raise BdbQuit
        finally:
            self.finalize()

Response: Run the IPython code using the IPython globals and locals of the cell.


Instruction:
Explain what the following Python 3 code does
def get_stack_data(self, frame, traceback, event_type):
        heap_data = Heap(self.options)
        stack_data = StackFrames(self.options)
        stack_frames, cur_frame_ind = self.get_stack(frame, traceback)

        for frame_ind, (frame, lineno) in enumerate(stack_frames):
            skip_this_stack = False

            # Skip the self.run calling frame (first frame)
            if frame_ind == 0:
                continue

            # Skip stack after a certain stack frame depth
            if len(stack_data) > self.options.depth:
                skip_this_stack = True
                break

            # Skip stack when frames dont belong to the current notebook or
            # current cell, I.e. frames in another global scope altogether
            # or frames in other cells
            if (not self.is_notebook_frame(frame) or
                    self.is_other_cell_frame(frame)):
                if not self.options.step_all:
                    skip_this_stack = True
                    break
                lineno = 0  # So line markers dont display for these frames
            else:
                lineno += 1  # Because cell magic is actually line 1

            # Filter out ignored names from the frame locals
            user_locals = filter_dict(
                frame.f_locals,
                ignore_vars + list(self.ipy_shell.user_ns_hidden.keys())
            )

            # Add frame and heap data
            stack_data.add(frame, lineno, event_type, user_locals)
            heap_data.add(user_locals)

        if not skip_this_stack and not stack_data.is_empty():
            self.trace_history.append(
                    stack_data,
                    heap_data,
                    self.stdout.getvalue()
            )

Response: Get the stack frames data at each of the hooks above ( Ie. for each iteration of the Python code.


Instruction:
Make a summary of the following Python 3 code
def filter_dict(d, exclude):
    ret = {}
    for key, value in d.items():
        if key not in exclude:
            ret.update({key: value})
    return ret

Response:
Returns a new dict with specified keys excluded from the origional dict
CTYPE


Instruction:
Write a Python 3 script to
redirect the stdout to use instead of the stdout.

Response:
def redirect_stdout(new_stdout):
    """Redirect the stdout

    Args:
        new_stdout (io.StringIO): New stdout to use instead
    """
    old_stdout, sys.stdout = sys.stdout, new_stdout
    try:
        yield None
    finally:
        sys.stdout = old_stdout


Instruction:
Can you generate a brief explanation for the following Python 3 code
def format(obj, options):
    formatters = {
        float_types: lambda x: '{:.{}g}'.format(x, options.digits),
    }
    for _types, fmtr in formatters.items():
        if isinstance(obj, _types):
            return fmtr(obj)
    try:
        if six.PY2 and isinstance(obj, six.string_types):
            return str(obj.encode('utf-8'))
        return str(obj)
    except:
        return 'OBJECT'

Response: Return a string representation of the Python object obj.


Instruction:
Given the following Python 3 function, write the documentation
def get_type_info(obj):
    if isinstance(obj, primitive_types):
        return ('primitive', type(obj).__name__)
    if isinstance(obj, sequence_types):
        return ('sequence', type(obj).__name__)
    if isinstance(obj, array_types):
        return ('array', type(obj).__name__)
    if isinstance(obj, key_value_types):
        return ('key-value', type(obj).__name__)
    if isinstance(obj, types.ModuleType):
        return ('module', type(obj).__name__)
    if isinstance(obj, (types.FunctionType, types.MethodType)):
        return ('function', type(obj).__name__)
    if isinstance(obj, type):
        if hasattr(obj, '__dict__'):
            return ('class', obj.__name__)
    if isinstance(type(obj), type):
        if hasattr(obj, '__dict__'):
            cls_name = type(obj).__name__
            if cls_name == 'classobj':
                cls_name = obj.__name__
                return ('class', '{}'.format(cls_name))
            if cls_name == 'instance':
                cls_name = obj.__class__.__name__
            return ('instance', '{} instance'.format(cls_name))

    return ('unknown', type(obj).__name__)

Response:
Get type information for a Python object
Taxonomy


Instruction:
Can you generate a brief explanation for the following Python 3 code
def refresh(self):
        self.accounts = self.accounts or []
        idx = 0
        for _acc in self._backend.accounts():
            _acc.wallet = self
            try:
                if self.accounts[idx]:
                    continue
            except IndexError:
                pass
            self.accounts.append(_acc)
            idx += 1

Response: Reloads the wallet and its accounts.


Instruction:
Create a Python 3 function to
return private spend key.

Response:
def spend_key(self):
        """
        Returns private spend key. None if wallet is view-only.

        :rtype: str or None
        """
        key = self._backend.spend_key()
        if key == numbers.EMPTY_KEY:
            return None
        return key


Instruction:
How would you code a function in Python 3 to
create a new account and appends it to the wallet s account list and returns it.

Response:
def new_account(self, label=None):
        """
        Creates new account, appends it to the :class:`Wallet`'s account list and returns it.

        :param label: account label as `str`
        :rtype: :class:`Account`
        """
        acc, addr = self._backend.new_account(label=label)
        assert acc.index == len(self.accounts)
        self.accounts.append(acc)
        return acc


Instruction:
Can you write a function in Python 3 where it
returns the number of confirmations for given transaction or payment.

Response:
def confirmations(self, txn_or_pmt):
        """
        Returns the number of confirmations for given
        :class:`Transaction <monero.transaction.Transaction>` or
        :class:`Payment <monero.transaction.Payment>` object.

        :rtype: int
        """
        if isinstance(txn_or_pmt, Payment):
            txn = txn_or_pmt.transaction
        else:
            txn = txn_or_pmt
        try:
            return max(0, self.height() - txn.height)
        except TypeError:
            return 0


Instruction:
Create a Python 3 function to
calculate the sub - address for the given account index and address index within the account.

Response:
def get_address(self, major, minor):
        """
        Calculates sub-address for account index (`major`) and address index within
        the account (`minor`).

        :rtype: :class:`BaseAddress <monero.address.BaseAddress>`
        """
        # ensure indexes are within uint32
        if major < 0 or major >= 2**32:
            raise ValueError('major index {} is outside uint32 range'.format(major))
        if minor < 0 or minor >= 2**32:
            raise ValueError('minor index {} is outside uint32 range'.format(minor))
        master_address = self.address()
        if major == minor == 0:
            return master_address
        master_svk = unhexlify(self.view_key())
        master_psk = unhexlify(self.address().spend_key())
        # m = Hs("SubAddr\0" || master_svk || major || minor)
        hsdata = b''.join([
                b'SubAddr\0', master_svk,
                struct.pack('<I', major), struct.pack('<I', minor)])
        m = keccak_256(hsdata).digest()
        # D = master_psk + m * B
        D = ed25519.add_compressed(
                ed25519.decodepoint(master_psk),
                ed25519.scalarmult(ed25519.B, ed25519.decodeint(m)))
        # C = master_svk * D
        C = ed25519.scalarmult(D, ed25519.decodeint(master_svk))
        netbyte = bytearray([
                42 if master_address.is_mainnet() else \
                63 if master_address.is_testnet() else 36])
        data = netbyte + ed25519.encodepoint(D) + ed25519.encodepoint(C)
        checksum = keccak_256(data).digest()[:4]
        return address.SubAddress(base58.encode(hexlify(data + checksum)))


Instruction:
Implement a function in Python 3 to
send a transfer from the default account.

Response:
def transfer(self, address, amount,
            priority=prio.NORMAL, payment_id=None, unlock_time=0,
            relay=True):
        """
        Sends a transfer from the default account. Returns a list of resulting transactions.

        :param address: destination :class:`Address <monero.address.Address>` or subtype
        :param amount: amount to send
        :param priority: transaction priority, implies fee. The priority can be a number
                    from 1 to 4 (unimportant, normal, elevated, priority) or a constant
                    from `monero.prio`.
        :param payment_id: ID for the payment (must be None if
                        :class:`IntegratedAddress <monero.address.IntegratedAddress>`
                        is used as the destination)
        :param unlock_time: the extra unlock delay
        :param relay: if `True`, the wallet will relay the transaction(s) to the network
                        immediately; when `False`, it will only return the transaction(s)
                        so they might be broadcasted later
        :rtype: list of :class:`Transaction <monero.transaction.Transaction>`
        """
        return self.accounts[0].transfer(
                address,
                amount,
                priority=priority,
                payment_id=payment_id,
                unlock_time=unlock_time,
                relay=relay)


Instruction:
Make a summary of the following Python 3 code
def transfer_multiple(self, destinations,
            priority=prio.NORMAL, payment_id=None, unlock_time=0,
            relay=True):
        """
        Sends a batch of transfers from the default account. Returns a list of resulting
        transactions.

        :param destinations: a list of destination and amount pairs: [(address, amount), ...]
        :param priority: transaction priority, implies fee. The priority can be a number
                    from 1 to 4 (unimportant, normal, elevated, priority) or a constant
                    from `monero.prio`.
        :param payment_id: ID for the payment (must be None if
                        :class:`IntegratedAddress <monero.address.IntegratedAddress>`
                        is used as a destination)
        :param unlock_time: the extra unlock delay
        :param relay: if `True`, the wallet will relay the transaction(s) to the network
                        immediately; when `False`, it will only return the transaction(s)
                        so they might be broadcasted later
        :rtype: list of :class:`Transaction <monero.transaction.Transaction>`
        """
        return self.accounts[0].transfer_multiple(
                destinations,
                priority=priority,
                payment_id=payment_id,
                unlock_time=unlock_time,
                relay=relay)

Response: Send a batch of transfers from the default account.


Instruction:
Make a summary of the following Python 3 code
def balance(self, unlocked=False):
        return self._backend.balances(account=self.index)[1 if unlocked else 0]

Response: Returns the current balance of the current account.


Instruction:
Create a Python 3 function to
create a new address.

Response:
def new_address(self, label=None):
        """
        Creates a new address.

        :param label: address label as `str`
        :rtype: :class:`SubAddress <monero.address.SubAddress>`
        """
        return self._backend.new_address(account=self.index, label=label)


Instruction:
Write a Python 3 script to
send a transfer to the specified address.

Response:
def transfer(self, address, amount,
            priority=prio.NORMAL, payment_id=None, unlock_time=0,
            relay=True):
        """
        Sends a transfer. Returns a list of resulting transactions.

        :param address: destination :class:`Address <monero.address.Address>` or subtype
        :param amount: amount to send
        :param priority: transaction priority, implies fee. The priority can be a number
                    from 1 to 4 (unimportant, normal, elevated, priority) or a constant
                    from `monero.prio`.
        :param payment_id: ID for the payment (must be None if
                        :class:`IntegratedAddress <monero.address.IntegratedAddress>`
                        is used as the destination)
        :param unlock_time: the extra unlock delay
        :param relay: if `True`, the wallet will relay the transaction(s) to the network
                        immediately; when `False`, it will only return the transaction(s)
                        so they might be broadcasted later
        :rtype: list of :class:`Transaction <monero.transaction.Transaction>`
        """
        return self._backend.transfer(
            [(address, amount)],
            priority,
            payment_id,
            unlock_time,
            account=self.index,
            relay=relay)


Instruction:
Given the following Python 3 function, write the documentation
def transfer_multiple(self, destinations,
            priority=prio.NORMAL, payment_id=None, unlock_time=0,
            relay=True):
        """
        Sends a batch of transfers. Returns a list of resulting transactions.

        :param destinations: a list of destination and amount pairs:
                    [(:class:`Address <monero.address.Address>`, `Decimal`), ...]
        :param priority: transaction priority, implies fee. The priority can be a number
                    from 1 to 4 (unimportant, normal, elevated, priority) or a constant
                    from `monero.prio`.
        :param payment_id: ID for the payment (must be None if
                        :class:`IntegratedAddress <monero.address.IntegratedAddress>`
                        is used as the destination)
        :param unlock_time: the extra unlock delay
        :param relay: if `True`, the wallet will relay the transaction(s) to the network
                        immediately; when `False`, it will only return the transaction(s)
                        so they might be broadcasted later
        :rtype: list of :class:`Transaction <monero.transaction.Transaction>`
        """
        return self._backend.transfer(
            destinations,
            priority,
            payment_id,
            unlock_time,
            account=self.index,
            relay=relay)

Response: Send a batch of transfers.


Instruction:
Can you write a function in Python 3 where it
converts Monero decimal to atomic integer of piconero.

Response:
def to_atomic(amount):
    """Convert Monero decimal to atomic integer of piconero."""
    if not isinstance(amount, (Decimal, float) + _integer_types):
        raise ValueError("Amount '{}' doesn't have numeric type. Only Decimal, int, long and "
                "float (not recommended) are accepted as amounts.")
    return int(amount * 10**12)


Instruction:
Write a Python 3 function that can
give a mnemonic word string confirm the computed checksum.

Response:
def _validate_checksum(self):
        """Given a mnemonic word string, confirm seed checksum (last word) matches the computed checksum.

        :rtype: bool
        """
        phrase = self.phrase.split(" ")
        if self.word_list.get_checksum(self.phrase) == phrase[-1]:
            return True
        raise ValueError("Invalid checksum")


Instruction:
Write a Python 3 script for
returning the master address represented by the seed.

Response:
def public_address(self, net='mainnet'):
        """Returns the master :class:`Address <monero.address.Address>` represented by the seed.

        :param net: the network, one of 'mainnet', 'testnet', 'stagenet'. Default is 'mainnet'.

        :rtype: :class:`Address <monero.address.Address>`
        """
        if net not in ('mainnet', 'testnet', 'stagenet'):
            raise ValueError(
                "Invalid net argument. Must be one of ('mainnet', 'testnet', 'stagenet').")
        netbyte = 18 if net == 'mainnet' else 53 if net == 'testnet' else 24
        data = "{:x}{:s}{:s}".format(netbyte, self.public_spend_key(), self.public_view_key())
        h = keccak_256()
        h.update(unhexlify(data))
        checksum = h.hexdigest()
        return address(base58.encode(data + checksum[0:8]))


Instruction:
Create a Python 3 function to
discover the proper class and return instance for a given Monero address.

Response:
def address(addr, label=None):
    """Discover the proper class and return instance for a given Monero address.

    :param addr: the address as a string-like object
    :param label: a label for the address (defaults to `None`)

    :rtype: :class:`Address`, :class:`SubAddress` or :class:`IntegratedAddress`
    """
    addr = str(addr)
    if _ADDR_REGEX.match(addr):
        netbyte = bytearray(unhexlify(base58.decode(addr)))[0]
        if netbyte in Address._valid_netbytes:
            return Address(addr, label=label)
        elif netbyte in SubAddress._valid_netbytes:
            return SubAddress(addr, label=label)
        raise ValueError("Invalid address netbyte {nb:x}. Allowed values are: {allowed}".format(
            nb=netbyte,
            allowed=", ".join(map(
                lambda b: '%02x' % b,
                sorted(Address._valid_netbytes + SubAddress._valid_netbytes)))))
    elif _IADDR_REGEX.match(addr):
        return IntegratedAddress(addr)
    raise ValueError("Address must be either 95 or 106 characters long base58-encoded string, "
        "is {addr} ({len} chars length)".format(addr=addr, len=len(addr)))


Instruction:
Write a Python 3 function that can
integrate the payment id into the address.

Response:
def with_payment_id(self, payment_id=0):
        """Integrates payment id into the address.

        :param payment_id: int, hexadecimal string or :class:`PaymentID <monero.numbers.PaymentID>`
                    (max 64-bit long)

        :rtype: `IntegratedAddress`
        :raises: `TypeError` if the payment id is too long
        """
        payment_id = numbers.PaymentID(payment_id)
        if not payment_id.is_short():
            raise TypeError("Payment ID {0} has more than 64 bits and cannot be integrated".format(payment_id))
        prefix = 54 if self.is_testnet() else 25 if self.is_stagenet() else 19
        data = bytearray([prefix]) + self._decoded[1:65] + struct.pack('>Q', int(payment_id))
        checksum = bytearray(keccak_256(data).digest()[:4])
        return IntegratedAddress(base58.encode(hexlify(data + checksum)))


Instruction:
Explain what the following Python 3 code does
def base_address(self):
        prefix = 53 if self.is_testnet() else 24 if self.is_stagenet() else 18
        data = bytearray([prefix]) + self._decoded[1:65]
        checksum = keccak_256(data).digest()[:4]
        return Address(base58.encode(hexlify(data + checksum)))

Response: Returns the base address without payment id.


Instruction:
Here you have a function in Python 3, explain what it does
def encode(hex):
    '''Encode hexadecimal string as base58 (ex: encoding a Monero address).'''
    data = _hexToBin(hex)
    l_data = len(data)

    if l_data == 0:
        return ""

    full_block_count = l_data // __fullBlockSize
    last_block_size = l_data % __fullBlockSize
    res_size = full_block_count * __fullEncodedBlockSize + __encodedBlockSizes[last_block_size]

    res = bytearray([__alphabet[0]] * res_size)

    for i in range(full_block_count):
        res = encode_block(data[(i*__fullBlockSize):(i*__fullBlockSize+__fullBlockSize)], res, i * __fullEncodedBlockSize)

    if last_block_size > 0:
        res = encode_block(data[(full_block_count*__fullBlockSize):(full_block_count*__fullBlockSize+last_block_size)], res, full_block_count * __fullEncodedBlockSize)

    return bytes(res).decode('ascii')

Response: Encode hexadecimal string as base58.


Instruction:
Implement a Python 3 function for
decoding a base58 string into hexidecimal form.

Response:
def decode(enc):
    '''Decode a base58 string (ex: a Monero address) into hexidecimal form.'''
    enc = bytearray(enc, encoding='ascii')
    l_enc = len(enc)

    if l_enc == 0:
        return ""

    full_block_count = l_enc // __fullEncodedBlockSize
    last_block_size = l_enc % __fullEncodedBlockSize
    try:
        last_block_decoded_size = __encodedBlockSizes.index(last_block_size)
    except ValueError:
        raise ValueError("Invalid encoded length: %d" % l_enc)

    data_size = full_block_count * __fullBlockSize + last_block_decoded_size

    data = bytearray(data_size)
    for i in range(full_block_count):
        data = decode_block(enc[(i*__fullEncodedBlockSize):(i*__fullEncodedBlockSize+__fullEncodedBlockSize)], data, i * __fullBlockSize)

    if last_block_size > 0:
        data = decode_block(enc[(full_block_count*__fullEncodedBlockSize):(full_block_count*__fullEncodedBlockSize+last_block_size)], data, full_block_count * __fullBlockSize)

    return _binToHex(data)


Instruction:
Write a Python 3 script for
converting hexadecimal string to mnemonic word representation with checksum.

Response:
def encode(cls, hex):
        """Convert hexadecimal string to mnemonic word representation with checksum.
        """
        out = []
        for i in range(len(hex) // 8):
            word = endian_swap(hex[8*i:8*i+8])
            x = int(word, 16)
            w1 = x % cls.n
            w2 = (x // cls.n + w1) % cls.n
            w3 = (x // cls.n // cls.n + w2) % cls.n
            out += [cls.word_list[w1], cls.word_list[w2], cls.word_list[w3]]
        checksum = cls.get_checksum(" ".join(out))
        out.append(checksum)
        return " ".join(out)


Instruction:
Given the following Python 3 function, write the documentation
def decode(cls, phrase):
        phrase = phrase.split(" ")
        out = ""
        for i in range(len(phrase) // 3):
            word1, word2, word3 = phrase[3*i:3*i+3]
            w1 = cls.word_list.index(word1)
            w2 = cls.word_list.index(word2) % cls.n
            w3 = cls.word_list.index(word3) % cls.n
            x = w1 + cls.n *((w2 - w1) % cls.n) + cls.n * cls.n * ((w3 - w2) % cls.n)
            out += endian_swap("%08x" % x)
        return out

Response: Decode a string into a hexadecimal representation of the class.


Instruction:
Implement a function in Python 3 to
give a mnemonic word string return a string of the computed checksum.

Response:
def get_checksum(cls, phrase):
        """Given a mnemonic word string, return a string of the computed checksum.

        :rtype: str
        """
        phrase_split = phrase.split(" ")
        if len(phrase_split) < 12:
            raise ValueError("Invalid mnemonic phrase")
        if len(phrase_split) > 13:
            # Standard format
            phrase = phrase_split[:24]
        else:
            # MyMonero format
            phrase = phrase_split[:12]
        wstr = "".join(word[:cls.unique_prefix_length] for word in phrase)
        wstr = bytearray(wstr.encode('utf-8'))
        z = ((crc32(wstr) & 0xffffffff) ^ 0xffffffff ) >> 0
        z2 = ((z ^ 0xffffffff) >> 0) % len(phrase)
        return phrase_split[z2]


Instruction:
Implement a function in Python 3 to
send a transaction to all peers.

Response:
def send_transaction(self, tx, relay=True):
        """
        Sends a transaction generated by a :class:`Wallet <monero.wallet.Wallet>`.

        :param tx: :class:`Transaction <monero.transaction.Transaction>`
        :param relay: whether to relay the transaction to peers. If `False`, the daemon will have
                to mine the transaction itself in order to have it included in the blockchain.
        """
        return self._backend.send_transaction(tx.blob, relay=relay)


Instruction:
Create a Python 3 function for
calling one in a while loop to allow user to pick many options. Returns a list of chosen options.

Response:
def many(prompt, *args, **kwargs):
    """Calls `pick` in a while loop to allow user to pick many
    options. Returns a list of chosen options.
    """
    def get_options(options, chosen):
        return [options[i] for i, c in enumerate(chosen) if c]

    def get_verbose_options(verbose_options, chosen):
        no, yes = ' ', '✔'
        if sys.version_info < (3, 3):
            no, yes = ' ', '@'
        opts = ['{} {}'.format(yes if c else no, verbose_options[i]) for i, c in enumerate(chosen)]
        return opts + ['{}{}'.format('  ', kwargs.get('done', 'done...'))]

    options, verbose_options = prepare_options(args)
    chosen = [False] * len(options)
    index = kwargs.get('idx', 0)

    default = kwargs.get('default', None)
    if isinstance(default, list):
        for idx in default:
            chosen[idx] = True
    if isinstance(default, int):
        chosen[default] = True

    while True:
        try:
            index = one(prompt, *get_verbose_options(verbose_options, chosen), return_index=True, idx=index)
        except QuestionnaireGoBack:
            if any(chosen):
                raise QuestionnaireGoBack(0)
            else:
                raise QuestionnaireGoBack
        if index == len(options):
            return get_options(options, chosen)
        chosen[index] = not chosen[index]


Instruction:
Can you generate a brief explanation for the following Python 3 code
def prepare_options(options):
    options_, verbose_options = [], []
    for option in options:
        if is_string(option):
            options_.append(option)
            verbose_options.append(option)
        else:
            options_.append(option[0])
            verbose_options.append(option[1])
    return options_, verbose_options

Response:
Create options and verbose options from strings and non - string iterables in
    options array.


Instruction:
How would you explain what the following Python 3 function does
def raw(prompt, *args, **kwargs):
    go_back = kwargs.get('go_back', '<')
    type_ = kwargs.get('type', str)
    default = kwargs.get('default', '')
    with stdout_redirected(sys.stderr):
        while True:
            try:
                if kwargs.get('secret', False):
                    answer = getpass.getpass(prompt)
                elif sys.version_info < (3, 0):
                    answer = raw_input(prompt)
                else:
                    answer = input(prompt)

                if not answer:
                    answer = default

                if answer == go_back:
                    raise QuestionnaireGoBack
                return type_(answer)
            except ValueError:
                eprint('\n`{}` is not a valid `{}`\n'.format(answer, type_))

Response: Calls raw input to allow user to input an arbitrary string.


Instruction:
Can you tell what is the following Python 3 function doing
def stdout_redirected(to):
    stdout = sys.stdout

    stdout_fd = fileno(stdout)
    # copy stdout_fd before it is overwritten
    with os.fdopen(os.dup(stdout_fd), 'wb') as copied:
        stdout.flush()  # flush library buffers that dup2 knows nothing about
        try:
            os.dup2(fileno(to), stdout_fd)  # $ exec >&to
        except ValueError:  # filename
            with open(to, 'wb') as to_file:
                os.dup2(to_file.fileno(), stdout_fd)  # $ exec > to
        try:
            yield stdout  # allow code to be run with the redirected stdout
        finally:
            # restore stdout to its previous value
            stdout.flush()
            os.dup2(copied.fileno(), stdout_fd)

Response: A context manager that redirects stdout to a file.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def get_operator(self, op):
        if op in self.OPERATORS:
            return self.OPERATORS.get(op)
        try:
            n_args = len(inspect.getargspec(op)[0])
            if n_args != 2:
                raise TypeError
        except:
            eprint('Error: invalid operator function. Operators must accept two args.')
            raise
        else:
            return op

Response: Assigns function to the operators property of the instance.


Instruction:
How would you code a function in Python 3 to
assign a prompter to this question.

Response:
def assign_prompter(self, prompter):
        """If you want to change the core prompters registry, you can
        override this method in a Question subclass.
        """
        if is_string(prompter):
            if prompter not in prompters:
                eprint("Error: '{}' is not a core prompter".format(prompter))
                sys.exit()
            self.prompter = prompters[prompter]
        else:
            self.prompter = prompter


Instruction:
Make a summary of the following Python 3 code
def add(self, *args, **kwargs):
        if 'question' in kwargs and isinstance(kwargs['question'], Question):
            question = kwargs['question']
        else:
            question = Question(*args, **kwargs)
        self.questions.setdefault(question.key, []).append(question)
        return question

Response: Add a Question instance to the questions dict.


Instruction:
Here you have a function in Python 3, explain what it does
def ask(self, error=None):
        q = self.next_question
        if q is None:
            return

        try:
            answer = q.prompter(self.get_prompt(q, error), *q.prompter_args, **q.prompter_kwargs)
        except QuestionnaireGoBack as e:
            steps = e.args[0] if e.args else 1
            if steps == 0:
                self.ask()  # user can redo current question even if `can_go_back` is `False`
                return
            self.go_back(steps)
        else:
            if q._validate:
                error = q._validate(answer)
                if error:
                    self.ask(error)
                    return
            if q._transform:
                answer = q._transform(answer)
            self.answers[q.key] = answer
            return answer

Response: Asks the next question in the questionnaire and returns the answer. If user goes back the questionnaire will go back and return the answer.


Instruction:
Can you create a Python 3 function that
returns the next question in the questionnaire or None if there are no questions left.

Response:
def next_question(self):
        """Returns the next `Question` in the questionnaire, or `None` if there
        are no questions left. Returns first question for whose key there is no
        answer and for which condition is satisfied, or for which there is no
        condition.
        """
        for key, questions in self.questions.items():
            if key in self.answers:
                continue
            for question in questions:
                if self.check_condition(question._condition):
                    return question
        return None


Instruction:
Write a Python 3 script to
move n questions back in the questionnaire by removing the last n answers.

Response:
def go_back(self, n=1):
        """Move `n` questions back in the questionnaire by removing the last `n`
        answers.
        """
        if not self.can_go_back:
            return
        N = max(len(self.answers)-abs(n), 0)
        self.answers = OrderedDict(islice(self.answers.items(), N))


Instruction:
Make a summary of the following Python 3 code
def format_answers(self, fmt='obj'):
        fmts = ('obj', 'array', 'plain')
        if fmt not in fmts:
            eprint("Error: '{}' not in {}".format(fmt, fmts))
            return

        def stringify(val):
            if type(val) in (list, tuple):
                return ', '.join(str(e) for e in val)
            return val

        if fmt == 'obj':
            return json.dumps(self.answers)
        elif fmt == 'array':
            answers = [[k, v] for k, v in self.answers.items()]
            return json.dumps(answers)
        elif fmt == 'plain':
            answers = '\n'.join('{}: {}'.format(k, stringify(v)) for k, v in self.answers.items())
            return answers

Response: Formats the answers depending on fmt.


Instruction:
Create a Python 3 function for
adding a new intent to the cache

Response:
def add_intent(self, name, lines, reload_cache=False):
        """
        Creates a new intent, optionally checking the cache first

        Args:
            name (str): The associated name of the intent
            lines (list<str>): All the sentences that should activate the intent
            reload_cache: Whether to ignore cached intent if exists
        """
        self.intents.add(name, lines, reload_cache)
        self.padaos.add_intent(name, lines)
        self.must_train = True


Instruction:
Explain what the following Python 3 code does
def add_entity(self, name, lines, reload_cache=False):
        Entity.verify_name(name)
        self.entities.add(Entity.wrap_name(name), lines, reload_cache)
        self.padaos.add_entity(name, lines)
        self.must_train = True

Response: Adds an entity that matches the given lines to the internal list of entities.


Instruction:
Can you tell what is the following Python 3 function doing
def load_entity(self, name, file_name, reload_cache=False):
        Entity.verify_name(name)
        self.entities.load(Entity.wrap_name(name), file_name, reload_cache)
        with open(file_name) as f:
            self.padaos.add_entity(name, f.read().split('\n'))
        self.must_train = True

Response: Loads an entity from a file.


Instruction:
Can you implement a function in Python 3 that
loads an intent from a file and adds it to the cache

Response:
def load_intent(self, name, file_name, reload_cache=False):
        """
        Loads an intent, optionally checking the cache first

        Args:
            name (str): The associated name of the intent
            file_name (str): The location of the intent file
            reload_cache (bool): Whether to refresh all of cache
        """
        self.intents.load(name, file_name, reload_cache)
        with open(file_name) as f:
            self.padaos.add_intent(name, f.read().split('\n'))
        self.must_train = True


Instruction:
Can you write a function in Python 3 where it
unloads an intent from the cache

Response:
def remove_intent(self, name):
        """Unload an intent"""
        self.intents.remove(name)
        self.padaos.remove_intent(name)
        self.must_train = True


Instruction:
Write a Python 3 function for
unloading an entity from the cache.

Response:
def remove_entity(self, name):
        """Unload an entity"""
        self.entities.remove(name)
        self.padaos.remove_entity(name)


Instruction:
Can you write a function in Python 3 where it
trains all the loaded intents that need to be updated from file

Response:
def train(self, debug=True, force=False, single_thread=False, timeout=20):
        """
        Trains all the loaded intents that need to be updated
        If a cache file exists with the same hash as the intent file,
        the intent will not be trained and just loaded from file

        Args:
            debug (bool): Whether to print a message to stdout each time a new intent is trained
            force (bool): Whether to force training if already finished
            single_thread (bool): Whether to force running in a single thread
            timeout (float): Seconds before cancelling training
        Returns:
            bool: True if training succeeded without timeout
        """
        if not self.must_train and not force:
            return
        self.padaos.compile()
        self.train_thread = Thread(target=self._train, kwargs=dict(
            debug=debug,
            single_thread=single_thread,
            timeout=timeout
        ), daemon=True)
        self.train_thread.start()
        self.train_thread.join(timeout)

        self.must_train = False
        return not self.train_thread.is_alive()


Instruction:
How would you implement a function in Python 3 that
trains in a subprocess which provides a timeout guarantees everything shuts down properly

Response:
def train_subprocess(self, *args, **kwargs):
        """
        Trains in a subprocess which provides a timeout guarantees everything shuts down properly

        Args:
            See <train>
        Returns:
            bool: True for success, False if timed out
        """
        ret = call([
            sys.executable, '-m', 'padatious', 'train', self.cache_dir,
            '-d', json.dumps(self.serialized_args),
            '-a', json.dumps(args),
            '-k', json.dumps(kwargs),
        ])
        if ret == 2:
            raise TypeError('Invalid train arguments: {} {}'.format(args, kwargs))
        data = self.serialized_args
        self.clear()
        self.apply_training_args(data)
        self.padaos.compile()
        if ret == 0:
            self.must_train = False
            return True
        elif ret == 10:  # timeout
            return False
        else:
            raise ValueError('Training failed and returned code: {}'.format(ret))


Instruction:
How would you explain what the following Python 3 function does
def calc_intents(self, query):
        if self.must_train:
            self.train()
        intents = {} if self.train_thread and self.train_thread.is_alive() else {
            i.name: i for i in self.intents.calc_intents(query, self.entities)
        }
        sent = tokenize(query)
        for perfect_match in self.padaos.calc_intents(query):
            name = perfect_match['name']
            intents[name] = MatchData(name, sent, matches=perfect_match['entities'], conf=1.0)
        return list(intents.values())

Response: Tests all the intents against the query and returns the data on how well each one matched against the query and returns the data on how well each one matched against the query


Instruction:
Can you create a Python 3 function that
tests all the intents against the query and returns the best intent

Response:
def calc_intent(self, query):
        """
        Tests all the intents against the query and returns
        match data of the best intent

        Args:
            query (str): Input sentence to test against intents
        Returns:
            MatchData: Best intent match
        """
        matches = self.calc_intents(query)
        if len(matches) == 0:
            return MatchData('', '')
        best_match = max(matches, key=lambda x: x.conf)
        best_matches = (match for match in matches if match.conf == best_match.conf)
        return min(best_matches, key=lambda x: sum(map(len, x.matches.values())))


Instruction:
Can you write a function in Python 3 where it
returns a list of all sub - sentences expanded in the order of the time they were created.

Response:
def expand(self):
        """
        Creates a combination of all sub-sentences.

        Returns:
            List<List<str>>: A list with all subsentence expansions combined in
                                every possible way
        """
        old_expanded = [[]]
        for sub in self._tree:
            sub_expanded = sub.expand()
            new_expanded = []
            while len(old_expanded) > 0:
                sentence = old_expanded.pop()
                for new in sub_expanded:
                    new_expanded.append(sentence + new)
            old_expanded = new_expanded
        return old_expanded


Instruction:
Create a Python 3 function to
return all of the options as seperated sub - sentences.

Response:
def expand(self):
        """
        Returns all of its options as seperated sub-sentences.

        Returns:
            List<List<str>>: A list containing the sentences created by all
                                expansions of its sub-sentences
        """
        options = []
        for option in self._tree:
            options.extend(option.expand())
        return options


Instruction:
Given the following Python 3 function, write the documentation
def _parse_expr(self):
        # List of all generated sentences
        sentence_list = []
        # Currently active sentence
        cur_sentence = []
        sentence_list.append(Sentence(cur_sentence))
        # Determine which form the current expression has
        while self._current_position < len(self.tokens):
            cur = self.tokens[self._current_position]
            self._current_position += 1
            if cur == '(':
                # Parse the subexpression
                subexpr = self._parse_expr()
                # Check if the subexpression only has one branch
                # -> If so, append "(" and ")" and add it as is
                normal_brackets = False
                if len(subexpr.tree()) == 1:
                    normal_brackets = True
                    cur_sentence.append(Word('('))
                # add it to the sentence
                cur_sentence.append(subexpr)
                if normal_brackets:
                    cur_sentence.append(Word(')'))
            elif cur == '|':
                # Begin parsing a new sentence
                cur_sentence = []
                sentence_list.append(Sentence(cur_sentence))
            elif cur == ')':
                # End parsing the current subexpression
                break
            # TODO anything special about {sth}?
            else:
                cur_sentence.append(Word(cur))
        return Options(sentence_list)

Response: Parse the expression and return a list of all the tokens that are in the current expression.


Instruction:
Can you generate the documentation for the following Python 3 function
def _train_and_save(obj, cache, data, print_updates):
    obj.train(data)
    if print_updates:
        print('Regenerated ' + obj.name + '.')
    obj.save(cache)

Response: Internal pickleable function used to train and save objects in another process


Instruction:
Write a Python 3 script for
wrapping SkillName into SkillName. entity into SkillName. entity

Response:
def wrap_name(name):
        """Wraps SkillName:entity into SkillName:{entity}"""
        if ':' in name:
            parts = name.split(':')
            intent_name, ent_name = parts[0], parts[1:]
            return intent_name + ':{' + ':'.join(ent_name) + '}'
        else:
            return '{' + name + '}'


Instruction:
Can you write a function in Python 3 where it
returns a unique id for the given list of lines

Response:
def lines_hash(lines):
    """
    Creates a unique binary id for the given lines
    Args:
        lines (list<str>): List of strings that should be collectively hashed
    Returns:
        bytearray: Binary hash
    """
    x = xxh32()
    for i in lines:
        x.update(i.encode())
    return x.digest()


Instruction:
Implement a Python 3 function for
taking a single sentence and returns a list of individual significant units ie. this is a sentence.

Response:
def tokenize(sentence):
    """
    Converts a single sentence into a list of individual significant units
    Args:
        sentence (str): Input string ie. 'This is a sentence.'
    Returns:
        list<str>: List of tokens ie. ['this', 'is', 'a', 'sentence']
    """
    tokens = []

    class Vars:
        start_pos = -1
        last_type = 'o'

    def update(c, i):
        if c.isalpha() or c in '-{}':
            t = 'a'
        elif c.isdigit() or c == '#':
            t = 'n'
        elif c.isspace():
            t = 's'
        else:
            t = 'o'

        if t != Vars.last_type or t == 'o':
            if Vars.start_pos >= 0:
                token = sentence[Vars.start_pos:i].lower()
                if token not in '.!?':
                    tokens.append(token)
            Vars.start_pos = -1 if t == 's' else i
        Vars.last_type = t

    for i, char in enumerate(sentence):
        update(char, i)
    update(' ', len(sentence))
    return tokens


Instruction:
Write a Python 3 function for
checking for duplicate inputs and sets the output to the max of the two outputs

Response:
def resolve_conflicts(inputs, outputs):
    """
    Checks for duplicate inputs and if there are any,
    remove one and set the output to the max of the two outputs
    Args:
        inputs (list<list<float>>): Array of input vectors
        outputs (list<list<float>>): Array of output vectors
    Returns:
        tuple<inputs, outputs>: The modified inputs and outputs
    """
    data = {}
    for inp, out in zip(inputs, outputs):
        tup = tuple(inp)
        if tup in data:
            data[tup].append(out)
        else:
            data[tup] = [out]

    inputs, outputs = [], []
    for inp, outs in data.items():
        inputs.append(list(inp))
        combined = [0] * len(outs[0])
        for i in range(len(combined)):
            combined[i] = max(j[i] for j in outs)
        outputs.append(combined)
    return inputs, outputs


Instruction:
How would you code a function in Python 3 to
re-apply type annotations from .pyi stubs to your codebase.

Response:
def main(src, pyi_dir, target_dir, incremental, quiet, replace_any, hg, traceback):
    """Re-apply type annotations from .pyi stubs to your codebase."""
    Config.incremental = incremental
    Config.replace_any = replace_any
    returncode = 0
    for src_entry in src:
        for file, error, exc_type, tb in retype_path(
            Path(src_entry),
            pyi_dir=Path(pyi_dir),
            targets=Path(target_dir),
            src_explicitly_given=True,
            quiet=quiet,
            hg=hg,
        ):
            print(f'error: {file}: {error}', file=sys.stderr)
            if traceback:
                print('Traceback (most recent call last):', file=sys.stderr)
                for line in tb:
                    print(line, file=sys.stderr, end='')
                print(f'{exc_type.__name__}: {error}', file=sys.stderr)
            returncode += 1
    if not src and not quiet:
        print('warning: no sources given', file=sys.stderr)

    # According to http://tldp.org/LDP/abs/html/index.html starting with 126
    # we have special returncodes.
    sys.exit(min(returncode, 125))


Instruction:
Can you tell what is the following Python 3 function doing
def retype_file(src, pyi_dir, targets, *, quiet=False, hg=False):
    with tokenize.open(src) as src_buffer:
        src_encoding = src_buffer.encoding
        src_node = lib2to3_parse(src_buffer.read())
    try:
        with open((pyi_dir / src.name).with_suffix('.pyi')) as pyi_file:
            pyi_txt = pyi_file.read()
    except FileNotFoundError:
        if not quiet:
            print(
                f'warning: .pyi file for source {src} not found in {pyi_dir}',
                file=sys.stderr,
            )
    else:
        pyi_ast = ast3.parse(pyi_txt)
        assert isinstance(pyi_ast, ast3.Module)
        reapply_all(pyi_ast.body, src_node)
    fix_remaining_type_comments(src_node)
    targets.mkdir(parents=True, exist_ok=True)
    with open(targets / src.name, 'w', encoding=src_encoding) as target_file:
        target_file.write(lib2to3_unparse(src_node, hg=hg))
    return targets / src.name

Response: Retype src finding types in pyi_dir and save in targets.


Instruction:
How would you explain what the following Python 3 function does
def lib2to3_parse(src_txt):
    grammar = pygram.python_grammar_no_print_statement
    drv = driver.Driver(grammar, pytree.convert)
    if src_txt[-1] != '\n':
        nl = '\r\n' if '\r\n' in src_txt[:1024] else '\n'
        src_txt += nl
    try:
        result = drv.parse_string(src_txt, True)
    except ParseError as pe:
        lineno, column = pe.context[1]
        lines = src_txt.splitlines()
        try:
            faulty_line = lines[lineno - 1]
        except IndexError:
            faulty_line = "<line number missing in source>"
        raise ValueError(f"Cannot parse: {lineno}:{column}: {faulty_line}") from None

    if isinstance(result, Leaf):
        result = Node(syms.file_input, [result])

    return result

Response: Given a string with source return the lib2to3 Node.


Instruction:
Write a Python 3 function for
giving a lib2to3 node return its string representation.

Response:
def lib2to3_unparse(node, *, hg=False):
    """Given a lib2to3 node, return its string representation."""
    code = str(node)
    if hg:
        from retype_hgext import apply_job_security
        code = apply_job_security(code)
    return code


Instruction:
Write a Python 3 function for
reapplying the typed_ast node into the lib2to3 tree.

Response:
def reapply_all(ast_node, lib2to3_node):
    """Reapplies the typed_ast node into the lib2to3 tree.

    Also does post-processing. This is done in reverse order to enable placing
    TypeVars and aliases that depend on one another.
    """
    late_processing = reapply(ast_node, lib2to3_node)
    for lazy_func in reversed(late_processing):
        lazy_func()


Instruction:
Can you generate the documentation for the following Python 3 function
def fix_remaining_type_comments(node):
    assert node.type == syms.file_input

    last_n = None
    for n in node.post_order():
        if last_n is not None:
            if n.type == token.NEWLINE and is_assignment(last_n):
                fix_variable_annotation_type_comment(n, last_n)
            elif n.type == syms.funcdef and last_n.type == syms.suite:
                fix_signature_annotation_type_comment(n, last_n, offset=1)
            elif n.type == syms.async_funcdef and last_n.type == syms.suite:
                fix_signature_annotation_type_comment(n, last_n, offset=2)
        last_n = n

Response: Converts type comments in node to proper annotated assignments.


Instruction:
Here you have a function in Python 3, explain what it does
def get_function_signature(fun, *, is_method=False):
    args = fun.args
    returns = fun.returns
    if fun.type_comment:
        try:
            args_tc, returns_tc = parse_signature_type_comment(fun.type_comment)
            if returns and returns_tc:
                raise ValueError(
                    "using both a type annotation and a type comment is not allowed"
                )
            returns = returns_tc
            copy_arguments_to_annotations(args, args_tc, is_method=is_method)
        except (SyntaxError, ValueError) as exc:
            raise ValueError(
                f"Annotation problem in function {fun.name!r}: " +
                f"{fun.lineno}:{fun.col_offset + 1}: {exc}"
            )
    copy_type_comments_to_annotations(args)

    return args, returns

Response: Returns the signature of a function.


Instruction:
Write a Python 3 function for
parsing the fugly signature type comment into AST nodes.

Response:
def parse_signature_type_comment(type_comment):
    """Parse the fugly signature type comment into AST nodes.

    Caveats: ASTifying **kwargs is impossible with the current grammar so we
    hack it into unary subtraction (to differentiate from Starred in vararg).

    For example from:
    "(str, int, *int, **Any) -> 'SomeReturnType'"

    To:
    ([ast3.Name, ast.Name, ast3.Name, ast.Name], ast3.Str)
    """
    try:
        result = ast3.parse(type_comment, '<func_type>', 'func_type')
    except SyntaxError:
        raise ValueError(f"invalid function signature type comment: {type_comment!r}")

    assert isinstance(result, ast3.FunctionType)
    if len(result.argtypes) == 1:
        argtypes = result.argtypes[0]
    else:
        argtypes = result.argtypes
    return argtypes, result.returns


Instruction:
Can you implement a function in Python 3 that
parses a type comment string into AST nodes.

Response:
def parse_type_comment(type_comment):
    """Parse a type comment string into AST nodes."""
    try:
        result = ast3.parse(type_comment, '<type_comment>', 'eval')
    except SyntaxError:
        raise ValueError(f"invalid type comment: {type_comment!r}") from None

    assert isinstance(result, ast3.Expression)
    return result.body


Instruction:
Can you generate a brief explanation for the following Python 3 code
def parse_arguments(arguments):
    arguments = f"def f{arguments}: ..."
    try:
        result = ast3.parse(arguments, '<arguments>', 'exec')
    except SyntaxError:
        raise ValueError(f"invalid arguments: {arguments!r}") from None

    assert isinstance(result, ast3.Module)
    assert len(result.body) == 1
    assert isinstance(result.body[0], ast3.FunctionDef)
    args = result.body[0].args
    copy_type_comments_to_annotations(args)
    return args

Response: Parse a string with function arguments into an AST node.


Instruction:
Implement a function in Python 3 to
copy AST nodes from type_comment into the ast3. arguments in args.

Response:
def copy_arguments_to_annotations(args, type_comment, *, is_method=False):
    """Copies AST nodes from `type_comment` into the ast3.arguments in `args`.

    Does validaation of argument count (allowing for untyped self/cls)
    and type (vararg and kwarg).
    """
    if isinstance(type_comment, ast3.Ellipsis):
        return

    expected = len(args.args)
    if args.vararg:
        expected += 1
    expected += len(args.kwonlyargs)
    if args.kwarg:
        expected += 1
    actual = len(type_comment) if isinstance(type_comment, list) else 1
    if expected != actual:
        if is_method and expected - actual == 1:
            pass  # fine, we're just skipping `self`, `cls`, etc.
        else:
            raise ValueError(
                f"number of arguments in type comment doesn't match; " +
                f"expected {expected}, found {actual}"
            )

    if isinstance(type_comment, list):
        next_value = type_comment.pop
    else:
        # If there's just one value, only one of the loops and ifs below will
        # be populated. We ensure this with the expected/actual length check
        # above.
        _tc = type_comment

        def next_value(index: int = 0) -> ast3.expr:
            return _tc

    for arg in args.args[expected - actual:]:
        ensure_no_annotation(arg.annotation)
        arg.annotation = next_value(0)

    if args.vararg:
        ensure_no_annotation(args.vararg.annotation)
        args.vararg.annotation = next_value(0)

    for arg in args.kwonlyargs:
        ensure_no_annotation(arg.annotation)
        arg.annotation = next_value(0)

    if args.kwarg:
        ensure_no_annotation(args.kwarg.annotation)
        args.kwarg.annotation = next_value(0)


Instruction:
Implement a Python 3 function for
copying argument type comments from the legacy long form to annotations in the entire function signature.

Response:
def copy_type_comments_to_annotations(args):
    """Copies argument type comments from the legacy long form to annotations
    in the entire function signature.
    """
    for arg in args.args:
        copy_type_comment_to_annotation(arg)

    if args.vararg:
        copy_type_comment_to_annotation(args.vararg)

    for arg in args.kwonlyargs:
        copy_type_comment_to_annotation(arg)

    if args.kwarg:
        copy_type_comment_to_annotation(args.kwarg)


Instruction:
Can you generate the documentation for the following Python 3 function
def maybe_replace_any_if_equal(name, expected, actual):
    is_equal = expected == actual
    if not is_equal and Config.replace_any:
        actual_str = minimize_whitespace(str(actual))
        if actual_str and actual_str[0] in {'"', "'"}:
            actual_str = actual_str[1:-1]
        is_equal = actual_str in {'Any', 'typing.Any', 't.Any'}

    if not is_equal:
        expected_annotation = minimize_whitespace(str(expected))
        actual_annotation = minimize_whitespace(str(actual))
        raise ValueError(
            f"incompatible existing {name}. " +
            f"Expected: {expected_annotation!r}, actual: {actual_annotation!r}"
        )

    return expected or actual

Response: Return the type given in expected and actual. Raise ValueError if expected is not equal to actual.


Instruction:
Write a Python 3 function that can
remove the legacy signature type comment leaving other comments if any.

Response:
def remove_function_signature_type_comment(body):
    """Removes the legacy signature type comment, leaving other comments if any."""
    for node in body.children:
        if node.type == token.INDENT:
            prefix = node.prefix.lstrip()
            if prefix.startswith('# type: '):
                node.prefix = '\n'.join(prefix.split('\n')[1:])
            break


Instruction:
Can you tell what is the following Python 3 function doing
def flatten_some(children):
    for node in children:
        if node.type in (syms.try_stmt, syms.suite):
            yield from flatten_some(node.children)
        else:
            yield node

Response: Generates nodes or leaves of a try or finally statement.


Instruction:
Here you have a function in Python 3, explain what it does
def pop_param(params):
    default = None

    name = params.pop(0)
    if name in (_star, _dstar):
        default = params.pop(0)
        if default == _comma:
            return name, default

    try:
        remainder = params.pop(0)
        if remainder == _eq:
            default = params.pop(0)
            remainder = params.pop(0)
        if remainder != _comma:
            raise ValueError(f"unexpected token: {remainder}")

    except IndexError:
        pass
    return name, default

Response: Pops the parameter and the remainder. Returns a tuple of name default or _star if the parameter is not present.


Instruction:
Implement a Python 3 function for
returning the offset after which a statement can be inserted to the body.

Response:
def get_offset_and_prefix(body, skip_assignments=False):
    """Returns the offset after which a statement can be inserted to the `body`.

    This offset is calculated to come after all imports, and maybe existing
    (possibly annotated) assignments if `skip_assignments` is True.

    Also returns the indentation prefix that should be applied to the inserted
    node.
    """
    assert body.type in (syms.file_input, syms.suite)

    _offset = 0
    prefix = ''
    for _offset, child in enumerate(body.children):
        if child.type == syms.simple_stmt:
            stmt = child.children[0]
            if stmt.type == syms.expr_stmt:
                expr = stmt.children
                if not skip_assignments:
                    break

                if (
                    len(expr) != 2 or
                    expr[0].type != token.NAME or
                    expr[1].type != syms.annassign or
                    _eq in expr[1].children
                ):
                    break

            elif stmt.type not in (syms.import_name, syms.import_from, token.STRING):
                break

        elif child.type == token.INDENT:
            assert isinstance(child, Leaf)
            prefix = child.value
        elif child.type != token.NEWLINE:
            break

    prefix, child.prefix = child.prefix, prefix
    return _offset, prefix


Instruction:
Can you tell what is the following Python 3 function doing
def fix_line_numbers(body):
    maxline = 0
    for node in body.pre_order():
        maxline += node.prefix.count('\n')
        if isinstance(node, Leaf):
            node.lineno = maxline
            maxline += str(node.value).count('\n')

Response: r Recomputes all line numbers based on the number of \ n characters.


Instruction:
Explain what the following Python 3 code does
def new(n, prefix=None):

    if isinstance(n, Leaf):
        return Leaf(n.type, n.value, prefix=n.prefix if prefix is None else prefix)

    # this is hacky, we assume complex nodes are just being reused once from the
    # original AST.
    n.parent = None
    if prefix is not None:
        n.prefix = prefix
    return n

Response: lib2to3 s AST requires unique objects as children.


Instruction:
Here you have a function in Python 3, explain what it does
def apply_job_security(code):
    buf = io.BytesIO(code.encode('utf8'))
    tokens = tokenize.tokenize(buf.readline)
    # NOTE: by setting the fullname to `mercurial.pycompat` below, we're
    # ensuring that hg-specific pycompat imports aren't inserted to the code.
    data = tokenize.untokenize(replacetokens(list(tokens), 'mercurial.pycompat'))
    return cast(str, data.decode('utf8'))

Response: Treat input code like Python 2.


Instruction:
How would you explain what the following Python 3 function does
def _load_info(self):
        '''Get user info for GBDX S3, put into instance vars for convenience.

        Args:
            None.

        Returns:
            Dictionary with S3 access key, S3 secret key, S3 session token,
            user bucket and user prefix (dict).
        '''

        url = '%s/prefix?duration=36000' % self.base_url
        r = self.gbdx_connection.get(url)
        r.raise_for_status()
        return r.json()

Response: Get user info for GBDX S3 put into instance vars for convenience.


Instruction:
Here you have a function in Python 3, explain what it does
def download(self, location, local_dir='.'):
        '''Download content from bucket/prefix/location.
           Location can be a directory or a file (e.g., my_dir or my_dir/my_image.tif)
           If location is a directory, all files in the directory are
           downloaded. If it is a file, then that file is downloaded.

           Args:
               location (str): S3 location within prefix.
               local_dir (str): Local directory where file(s) will be stored.
                                Default is here.
        '''

        self.logger.debug('Getting S3 info')
        bucket = self.info['bucket']
        prefix = self.info['prefix']

        self.logger.debug('Connecting to S3')
        s3conn = self.client

        # remove head and/or trail backslash from location
        location = location.strip('/')

        self.logger.debug('Downloading contents')
        objects = s3conn.list_objects(Bucket=bucket, Prefix=(prefix+'/'+location))
        if 'Contents' not in objects:
            raise ValueError('Download target {}/{}/{} was not found or inaccessible.'.format(bucket, prefix, location))
        for s3key in objects['Contents']:
            key = s3key['Key']

            # skip directory keys
            if not key or key.endswith('/'):
                continue

            # get path to each file
            filepath = key.replace(prefix+'/'+location, '', 1).lstrip('/')
            filename = key.split('/')[-1]

            #self.logger.debug(filename)
            file_dir = filepath.split('/')[:-1]
            file_dir = '/'.join(file_dir)
            full_dir = os.path.join(local_dir, file_dir)

            # make sure directory exists
            if not os.path.isdir(full_dir):
                os.makedirs(full_dir)

            # download file
            s3conn.download_file(bucket, key, os.path.join(full_dir, filename))

        self.logger.debug('Done!')

Response: Download content from bucket / prefix / location.


Instruction:
Create a Python 3 function for
deleting content in bucket / prefix / location.

Response:
def delete(self, location):
        '''Delete content in bucket/prefix/location.
           Location can be a directory or a file (e.g., my_dir or my_dir/my_image.tif)
           If location is a directory, all files in the directory are deleted.
           If it is a file, then that file is deleted.

           Args:
               location (str): S3 location within prefix. Can be a directory or
                               a file (e.g., my_dir or my_dir/my_image.tif).
        '''
        bucket = self.info['bucket']
        prefix = self.info['prefix']

        self.logger.debug('Connecting to S3')
        s3conn = self.client

        # remove head and/or trail backslash from location
        if location[0] == '/':
            location = location[1:]
        if location[-1] == '/':
            location = location[:-2]

        self.logger.debug('Deleting contents')

        for s3key in s3conn.list_objects(Bucket=bucket, Prefix=(prefix+'/'+location))['Contents']:
            s3conn.delete_object(Bucket=bucket, Key=s3key['Key'])

        self.logger.debug('Done!')


Instruction:
Can you tell what is the following Python 3 function doing
def upload(self, local_file, s3_path=None):
        '''
        Upload files to your DG S3 bucket/prefix.

        Args:
            local_file (str): a path to a local file to upload, directory structures are not mirrored
            s3_path: a key (location) on s3 to upload the file to

        Returns:
            str: s3 path file was saved to

        Examples:
            >>> upload('path/to/image.tif')
            'mybucket/myprefix/image.tif'

            >>> upload('./images/image.tif')
            'mybucket/myprefix/image.tif'

            >>> upload('./images/image.tif', s3_path='images/image.tif')
            'mybucket/myprefix/images/image.tif'
        '''
        if not os.path.exists(local_file):
            raise Exception(local_file + " does not exist.")

        if s3_path is None:
            s3_path = os.path.basename(local_file)

        bucket = self.info['bucket']
        prefix = self.info['prefix']

        self.logger.debug('Connecting to S3')
        s3conn = self.client
        self.logger.debug('Uploading file {}'.format(local_file))
        s3conn.upload_file(local_file, bucket, prefix+'/'+s3_path)
        self.logger.debug('Done!')
        return '{}/{}/{}'.format(bucket, prefix, s3_path)

Response: Uploads a local file to the DG S3 bucket and returns the path to the file uploaded to the DG S3 bucket.


Instruction:
Make a summary of the following Python 3 code
def rgb(self, **kwargs):
        ''' Convert the image to a 3 band RGB for plotting

        This method shares the same arguments as plot(). It will perform visual adjustment on the
        image and prepare the data for plotting in MatplotLib. Values are converted to an
        appropriate precision and the axis order is changed to put the band axis last.
        '''

        if "bands" in kwargs:
            use_bands = kwargs["bands"]
            assert len(use_bands) == 3, 'Plot method only supports single or 3-band outputs'
            del kwargs["bands"]
        else:
            use_bands = self._rgb_bands
        if kwargs.get('blm') == True:
            return self.histogram_match(use_bands, **kwargs)
        # if not specified or DRA'ed, default to a 2-98 stretch
        if "histogram" not in kwargs:
            if "stretch" not in kwargs:
                if not self.options.get('dra'):
                    kwargs['stretch'] = [2,98]
            return self.histogram_stretch(use_bands, **kwargs)
        elif kwargs["histogram"] == "equalize":
            return self.histogram_equalize(use_bands, **kwargs)
        elif kwargs["histogram"] == "match":
            return self.histogram_match(use_bands, **kwargs)
        elif kwargs["histogram"] == "minmax":
            return self.histogram_stretch(use_bands, stretch=[0, 100], **kwargs)
        # DRA'ed images should be left alone if not explicitly adjusted
        elif kwargs["histogram"] == "ignore" or self.options.get('dra'):
            data = self._read(self[use_bands,...], **kwargs)
            return np.rollaxis(data, 0, 3)
        else:
            raise KeyError('Unknown histogram parameter, use "equalize", "match", "minmax", or "ignore"')

Response: Convert the image to a 3 band RGB for plotting


Instruction:
How would you implement a function in Python 3 that
equalizes and the histogram and normalize value range

Response:
def histogram_equalize(self, use_bands, **kwargs):
        ''' Equalize and the histogram and normalize value range
            Equalization is on all three bands, not per-band'''
        data = self._read(self[use_bands,...], **kwargs)
        data = np.rollaxis(data.astype(np.float32), 0, 3)
        flattened = data.flatten()
        if 0 in data:
            masked = np.ma.masked_values(data, 0).compressed()
            image_histogram, bin_edges = np.histogram(masked, 256)
        else:
            image_histogram, bin_edges = np.histogram(flattened, 256)
        bins = (bin_edges[:-1] + bin_edges[1:]) / 2.0
        cdf = image_histogram.cumsum()
        cdf = cdf / float(cdf[-1])
        image_equalized = np.interp(flattened, bins, cdf).reshape(data.shape)
        if 'stretch' in kwargs or 'gamma' in kwargs:
            return self._histogram_stretch(image_equalized, **kwargs)
        else:
            return image_equalized


Instruction:
Here you have a function in Python 3, explain what it does
def histogram_match(self, use_bands, blm_source=None, **kwargs):
        ''' Match the histogram to existing imagery '''
        assert has_rio, "To match image histograms please install rio_hist"
        data = self._read(self[use_bands,...], **kwargs)
        data = np.rollaxis(data.astype(np.float32), 0, 3)
        if 0 in data:
            data = np.ma.masked_values(data, 0)
        bounds = self._reproject(box(*self.bounds), from_proj=self.proj, to_proj="EPSG:4326").bounds
        if blm_source == 'browse':
            from gbdxtools.images.browse_image import BrowseImage
            ref = BrowseImage(self.cat_id, bbox=bounds).read()
        else:
            from gbdxtools.images.tms_image import TmsImage
            tms = TmsImage(zoom=self._calc_tms_zoom(self.affine[0]), bbox=bounds, **kwargs)
            ref = np.rollaxis(tms.read(), 0, 3)
        out = np.dstack([rio_match(data[:,:,idx], ref[:,:,idx].astype(np.double)/255.0)
                        for idx in range(data.shape[-1])])
        if 'stretch' in kwargs or 'gamma' in kwargs:
            return self._histogram_stretch(out, **kwargs)
        else:
            return out

Response: Match the histogram to existing imagery


Instruction:
Can you generate the documentation for the following Python 3 function
def _histogram_stretch(self, data, **kwargs):
        ''' perform a contrast stretch and/or gamma adjustment '''
        limits = {}
        # get the image min-max statistics
        for x in range(3):
            band = data[:,:,x]
            try:
                limits[x] = np.percentile(band, kwargs.get("stretch", [0,100]))
            except IndexError:
                # this band has no dynamic range and cannot be stretched
                return data
        # compute the stretch
        for x in range(3):
            band = data[:,:,x]
            if 0 in band:
                band = np.ma.masked_values(band, 0).compressed()
            top = limits[x][1]
            bottom = limits[x][0]
            if top != bottom: # catch divide by zero
                data[:,:,x] = (data[:,:,x] - bottom) / float(top - bottom) * 255.0
        data = np.clip(data, 0, 255).astype("uint8")
        # gamma adjust
        if "gamma" in kwargs:
            invGamma = 1.0 / kwargs['gamma']
            lut = np.array([((i / 255.0) ** invGamma) * 255
		            for i in np.arange(0, 256)]).astype("uint8")
            data = np.take(lut, data)
        return data

Response: perform a contrast stretch and gamma adjustment


Instruction:
Given the following Python 3 function, write the documentation
def ndvi(self, **kwargs):
        data = self._read(self[self._ndvi_bands,...]).astype(np.float32)
        return (data[0,:,:] - data[1,:,:]) / (data[0,:,:] + data[1,:,:])

Response: Calculates Normalized Difference Vegetation Index using NIR and Red of an image.


Instruction:
How would you explain what the following Python 3 function does
def ndwi(self):
        data = self._read(self[self._ndwi_bands,...]).astype(np.float32)
        return (data[1,:,:] - data[0,:,:]) / (data[0,:,:] + data[1,:,:])

Response: Calculates Normalized Difference Water Index using Coastal and NIR2 bands for Landsat8 and sentinel2 calculated by using Green and NIR2 bands for Landsat8 and sentinel2.


Instruction:
Write a Python 3 function that can
plot the image with MatplotLib and return the image.

Response:
def plot(self, spec="rgb", **kwargs):
        ''' Plot the image with MatplotLib

        Plot sizing includes default borders and spacing. If the image is shown in Jupyter the outside whitespace will be automatically cropped to save size, resulting in a smaller sized image than expected.

        Histogram options:
            * 'equalize': performs histogram equalization on the image.
            * 'minmax': stretch the pixel range to the minimum and maximum input pixel values. Equivalent to stretch=[0,100].
            * 'match': match the histogram to the Maps API imagery. Pass the additional keyword blm_source='browse' to match to the Browse Service (image thumbnail) instead.
            * 'ignore': Skip dynamic range adjustment, in the event the image is already correctly balanced and the values are in the correct range.

        Gamma values greater than 1 will brighten the image midtones, values less than 1 will darken the midtones.

        Plots generated with the histogram options of 'match' and 'equalize' can be combined with the stretch and gamma options. The stretch and gamma adjustments will be applied after the histogram adjustments.

        Args:
            w (float or int): width of plot in inches at 72 dpi, default is 10
            h (float or int): height of plot in inches at 72 dpi, default is 10
            title (str): Title to use on the plot
            fontsize (int): Size of title font, default is 22. Size is measured in points.
            bands (list): bands to use for plotting, such as bands=[4,2,1]. Defaults to the image's natural RGB bands. This option is useful for generating pseudocolor images when passed a list of three bands. If only a single band is provided, a colormapped plot will be generated instead.
            cmap (str): MatPlotLib colormap name to use for single band images. Default is colormap='Grey_R'.
            histogram (str): either 'equalize', 'minmax', 'match', or ignore
            stretch (list): stretch the histogram between two percentile values, default is [2,98]
            gamma (float): adjust image gamma, default is 1.0
        '''

        if self.shape[0] == 1 or ("bands" in kwargs and len(kwargs["bands"]) == 1):
            if "cmap" in kwargs:
                cmap = kwargs["cmap"]
                del kwargs["cmap"]
            else:
                cmap = "Greys_r"
            self._plot(tfm=self._single_band, cmap=cmap, **kwargs)
        else:
            if spec == "rgb" and self._has_token(**kwargs):
                self._plot(tfm=self.rgb, **kwargs)
            else:
                self._plot(tfm=getattr(self, spec), **kwargs)


Instruction:
Given the following Python 3 function, write the documentation
def get_images_by_catid_and_aoi(self, catid, aoi_wkt):

        self.logger.debug('Retrieving IDAHO metadata')

        # use the footprint to get the IDAHO id
        url = '%s/search' % self.base_url

        body = {"filters": ["catalogID = '%s'" % catid],
                "types": ["IDAHOImage"],
                "searchAreaWkt": aoi_wkt}

        r = self.gbdx_connection.post(url, data=json.dumps(body))

        r.raise_for_status()
        if r.status_code == 200:
            results = r.json()
            numresults = len(results['results'])
            self.logger.debug('%s IDAHO images found associated with catid %s'
                              % (numresults, catid))

            return results

Response: Get the IDAHO images associated with a given catalog ID and well - known text of the area of interest.