Instruction:
Explain what the following Python 3 code does
def get_docker_secret(name, default=None, cast_to=str, autocast_name=True, getenv=True, safe=True,
                      secrets_dir=os.path.join(root, 'var', 'run', 'secrets')):
    """This function fetches a docker secret

    :param name: the name of the docker secret
    :param default: the default value if no secret found
    :param cast_to: casts the value to the given type
    :param autocast_name: whether the name should be lowercase for secrets and upper case for environment
    :param getenv: if environment variable should be fetched as fallback
    :param safe: Whether the function should raise exceptions
    :param secrets_dir: the directory where the secrets are stored
    :returns: docker secret or environment variable depending on params
    :raises TypeError: if cast fails due to wrong type (None)
    :raises ValueError: if casts fails due to Value
    """

    # cast name if autocast enabled
    name_secret = name.lower() if autocast_name else name
    name_env = name.upper() if autocast_name else name

    # initiallize value
    value = None

    # try to read from secret file
    try:
        with open(os.path.join(secrets_dir, name_secret), 'r') as secret_file:
            value = secret_file.read()
    except IOError as e:
        # try to read from env if enabled
        if getenv:
            value = os.environ.get(name_env)

    # set default value if no value found
    if value is None:
        value = default

    # try to cast
    try:
        # so None wont be cast to 'None'
        if value is None:
            raise TypeError('value is None')

        # special case bool
        if cast_to == bool:
            if value not in ('True', 'true', 'False', 'false'):
                raise ValueError('value %s not of type bool' % value)
            value = 1 if value in ('True', 'true') else 0

        # try to cast
        return cast_to(value)

    except (TypeError, ValueError) as e:
        # whether exception should be thrown
        if safe:
            return default
        raise e

Response: This function fetches a docker secret from the secrets directory.


Instruction:
Make a summary of the following Python 3 code
def _update_hash(self, arg):
        if isinstance(arg, np.ndarray):
            self.ahash.update(arg.view(np.uint8))
        elif isinstance(arg, list):
            [self._update_hash(a) for a in arg]
        else:
            self.ahash.update(str(arg).encode('utf-8'))

Response: Takes an argument and updates the hash.


Instruction:
How would you explain what the following Python 3 function does
def clear_cache():
        del Cache._keys
        for k in list(Cache._cache.keys()):
            it = Cache._cache.pop(k)
            del it
        del Cache._cache
        Cache._keys = []
        Cache._cache = {}
        gc.collect()

Response: Remove all cached objects


Instruction:
Here you have a function in Python 3, explain what it does
def get_pos(self, sector):
        if not self.sector_is_valid(sector):
            raise ByteNotInFile166("Sector %d out of range" % sector)
        pos = sector * self.sector_size + self.header_offset
        size = self.sector_size
        return pos, size

Response: Get the offset and size of the entry in the raw data of the disk image.


Instruction:
Write a Python 3 function that can
get a slice of the data from the specified sector.

Response:
def get_sector_slice(self, start, end=None):
        """ Get contiguous sectors

        :param start: first sector number to read (note: numbering starts from 1)
        :param end: last sector number to read
        :returns: bytes
        """
        pos, size = self.header.get_pos(start)
        if end is None:
            end = start
        while start < end:
            start += 1
            _, more = self.header.get_pos(start)
            size += more
        return slice(pos, pos + size)


Instruction:
Can you create a Python 3 function that
gets contiguous sectors in the cache.

Response:
def get_sectors(self, start, end=None):
        """ Get contiguous sectors

        :param start: first sector number to read (note: numbering starts from 1)
        :param end: last sector number to read
        :returns: bytes
        """
        s = self.get_sector_slice(start, end)
        return self.bytes[s], self.style[s]


Instruction:
Can you implement a function in Python 3 that
writes data to a file on disk.

Response:
def write_file(self, filename, filetype, data):
        """Write data to a file on disk

        This throws various exceptions on failures, for instance if there is
        not enough space on disk or a free entry is not available in the
        catalog.
        """
        state = self.begin_transaction()
        try:
            directory = self.directory_class(self.header)
            self.get_directory(directory)
            dirent = directory.add_dirent(filename, filetype)
            data = to_numpy(data)
            sector_list = self.build_sectors(data)
            vtoc = self.get_vtoc_object()
            directory.save_dirent(self, dirent, vtoc, sector_list)
            self.write_sector_list(sector_list)
            self.write_sector_list(vtoc)
            self.write_sector_list(directory)
        except errors.AtrError:
            self.rollback_transaction(state)
            raise
        finally:
            self.get_metadata()


Instruction:
Can you tell what is the following Python 3 function doing
def _apply_scale(self, a, scale, feat):
        if scale == "linear":
            b = a
        elif scale == "log":
            with warnings.catch_warnings(record=True) as w:
                warnings.simplefilter("always")
                b = np.log(a)
                if len(w):
                    # Tell the user that the log-transformation issued
                    # a warning.
                    warnings.warn("Invalid values encounterd in np.log "
                                  "while scaling feature '{}'!".format(feat))
        else:
            raise ValueError("`scale` must be either 'linear' or 'log', "
                             + "got '{}'!".format(scale))
        return b

Response: Helper function for transforming an array a ray to log - scale.


Instruction:
Can you tell what is the following Python 3 function doing
def features(self):
        mycols = []
        for col in dfn.feature_names:
            if col in self:
                mycols.append(col)
        mycols.sort()
        return mycols

Response: Return a list of all available features


Instruction:
How would you explain what the following Python 3 function does
def get_downsampled_scatter(self, xax="area_um", yax="deform",
                                downsample=0, xscale="linear",
                                yscale="linear"):
        """Downsampling by removing points at dense locations

        Parameters
        ----------
        xax: str
            Identifier for x axis (e.g. "area_um", "aspect", "deform")
        yax: str
            Identifier for y axis
        downsample: int
            Number of points to draw in the down-sampled plot.
            This number is either

            - >=1: exactly downsample to this number by randomly adding
                   or removing points
            - 0  : do not perform downsampling
        xscale: str
            If set to "log", take the logarithm of the x-values before
            performing downsampling. This is useful when data are are
            displayed on a log-scale. Defaults to "linear".
        yscale: str
            See `xscale`.

        Returns
        -------
        xnew, xnew: filtered x and y
        """
        if downsample < 0:
            raise ValueError("`downsample` must be zero or positive!")

        downsample = int(downsample)
        xax = xax.lower()
        yax = yax.lower()

        # Get data
        x = self[xax][self.filter.all]
        y = self[yax][self.filter.all]

        # Apply scale (no change for linear scale)
        xs = self._apply_scale(x, xscale, xax)
        ys = self._apply_scale(y, yscale, yax)

        _, _, idx = downsampling.downsample_grid(xs, ys,
                                                 samples=downsample,
                                                 ret_idx=True)
        self._plot_filter = idx
        return x[idx], y[idx]

Response: Downsampling of the data for a specific set of locations.


Instruction:
Can you write a function in Python 3 where it
evaluates the kernel density estimate for contour plots.

Response:
def get_kde_contour(self, xax="area_um", yax="deform", xacc=None,
                        yacc=None, kde_type="histogram", kde_kwargs={},
                        xscale="linear", yscale="linear"):
        """Evaluate the kernel density estimate for contour plots

        Parameters
        ----------
        xax: str
            Identifier for X axis (e.g. "area_um", "aspect", "deform")
        yax: str
            Identifier for Y axis
        xacc: float
            Contour accuracy in x direction
        yacc: float
            Contour accuracy in y direction
        kde_type: str
            The KDE method to use
        kde_kwargs: dict
            Additional keyword arguments to the KDE method
        xscale: str
            If set to "log", take the logarithm of the x-values before
            computing the KDE. This is useful when data are are
            displayed on a log-scale. Defaults to "linear".
        yscale: str
            See `xscale`.

        Returns
        -------
        X, Y, Z : coordinates
            The kernel density Z evaluated on a rectangular grid (X,Y).
        """
        xax = xax.lower()
        yax = yax.lower()
        kde_type = kde_type.lower()
        if kde_type not in kde_methods.methods:
            raise ValueError("Not a valid kde type: {}!".format(kde_type))

        # Get data
        x = self[xax][self.filter.all]
        y = self[yax][self.filter.all]

        # Apply scale (no change for linear scale)
        xs = self._apply_scale(x, xscale, xax)
        ys = self._apply_scale(y, yscale, yax)

        # accuracy (bin width) of KDE estimator
        if xacc is None:
            xacc = kde_methods.bin_width_doane(xs) / 5
        if yacc is None:
            yacc = kde_methods.bin_width_doane(ys) / 5

        # Ignore infs and nans
        bad = kde_methods.get_bad_vals(xs, ys)
        xc = xs[~bad]
        yc = ys[~bad]

        xnum = int(np.ceil((xc.max() - xc.min()) / xacc))
        ynum = int(np.ceil((yc.max() - yc.min()) / yacc))

        xlin = np.linspace(xc.min(), xc.max(), xnum, endpoint=True)
        ylin = np.linspace(yc.min(), yc.max(), ynum, endpoint=True)

        xmesh, ymesh = np.meshgrid(xlin, ylin, indexing="ij")

        kde_fct = kde_methods.methods[kde_type]
        if len(x):
            density = kde_fct(events_x=xs, events_y=ys,
                              xout=xmesh, yout=ymesh,
                              **kde_kwargs)
        else:
            density = []

        # Convert mesh back to linear scale if applicable
        if xscale == "log":
            xmesh = np.exp(xmesh)
        if yscale == "log":
            ymesh = np.exp(ymesh)

        return xmesh, ymesh, density


Instruction:
Can you tell what is the following Python 3 function doing
def get_kde_scatter(self, xax="area_um", yax="deform", positions=None,
                        kde_type="histogram", kde_kwargs={}, xscale="linear",
                        yscale="linear"):
        """Evaluate the kernel density estimate for scatter plots

        Parameters
        ----------
        xax: str
            Identifier for X axis (e.g. "area_um", "aspect", "deform")
        yax: str
            Identifier for Y axis
        positions: list of two 1d ndarrays or ndarray of shape (2, N)
            The positions where the KDE will be computed. Note that
            the KDE estimate is computed from the the points that
            are set in `self.filter.all`.
        kde_type: str
            The KDE method to use
        kde_kwargs: dict
            Additional keyword arguments to the KDE method
        xscale: str
            If set to "log", take the logarithm of the x-values before
            computing the KDE. This is useful when data are are
            displayed on a log-scale. Defaults to "linear".
        yscale: str
            See `xscale`.

        Returns
        -------
        density : 1d ndarray
            The kernel density evaluated for the filtered data points.
        """
        xax = xax.lower()
        yax = yax.lower()
        kde_type = kde_type.lower()
        if kde_type not in kde_methods.methods:
            raise ValueError("Not a valid kde type: {}!".format(kde_type))

        # Get data
        x = self[xax][self.filter.all]
        y = self[yax][self.filter.all]

        # Apply scale (no change for linear scale)
        xs = self._apply_scale(x, xscale, xax)
        ys = self._apply_scale(y, yscale, yax)

        if positions is None:
            posx = None
            posy = None
        else:
            posx = self._apply_scale(positions[0], xscale, xax)
            posy = self._apply_scale(positions[1], yscale, yax)

        kde_fct = kde_methods.methods[kde_type]
        if len(x):
            density = kde_fct(events_x=xs, events_y=ys,
                              xout=posx, yout=posy,
                              **kde_kwargs)
        else:
            density = []

        return density

Response: Evaluate the kernel density estimate for scatter plots.


Instruction:
How would you code a function in Python 3 to
associate a Polygon Filter with this instance

Response:
def polygon_filter_add(self, filt):
        """Associate a Polygon Filter with this instance

        Parameters
        ----------
        filt: int or instance of `PolygonFilter`
            The polygon filter to add
        """
        if not isinstance(filt, (PolygonFilter, int, float)):
            msg = "`filt` must be a number or instance of PolygonFilter!"
            raise ValueError(msg)

        if isinstance(filt, PolygonFilter):
            uid = filt.unique_id
        else:
            uid = int(filt)
        # append item
        self.config["filtering"]["polygon filters"].append(uid)


Instruction:
Can you generate the documentation for the following Python 3 function
def polygon_filter_rm(self, filt):
        if not isinstance(filt, (PolygonFilter, int, float)):
            msg = "`filt` must be a number or instance of PolygonFilter!"
            raise ValueError(msg)

        if isinstance(filt, PolygonFilter):
            uid = filt.unique_id
        else:
            uid = int(filt)
        # remove item
        self.config["filtering"]["polygon filters"].remove(uid)

Response: Removes a polygon filter from this instance.


Instruction:
Make a summary of the following Python 3 code
def check_dataset(path_or_ds):
    aler = []
    info = []
    viol = []
    if isinstance(path_or_ds, RTDCBase):
        ds = path_or_ds
    else:
        ds = load_file(path_or_ds)
    # check for meta data types
    for sec in ds.config:
        for key in ds.config[sec]:
            if sec in dfn.CFG_ANALYSIS:
                # TODO:
                # - properly test against analysis keywords
                #   (filtering, calculation)
                pass
            elif (sec not in dfn.config_keys or
                  key not in dfn.config_keys[sec]):
                viol.append("Metadata: Unknown key [{}] '{}'".format(sec, key))
            elif not isinstance(ds.config[sec][key],
                                dfn.config_types[sec][key]):
                viol.append("Metadata: Datatype of [{}] '{}'".format(sec, key)
                            + "must be '{}'".format(dfn.config_types[sec][key])
                            )
    # check existence of meta data keys
    # These "must" be present:
    tocheck = IMPORTANT_KEYS
    # These sections "should" be fully present
    tocheck_sec_aler = ["experiment", "imaging", "online_contour", "setup"]
    # should we also check for fluorescence keys?
    if ("fluorescence" in ds.config or
        "fl1_max" in ds._events or
        "fl2_max" in ds._events or
            "fl3_max" in ds._events):
        info.append("Fluorescence: True")
        tocheck = tocheck.copy()
        tocheck.update(IMPORTANT_KEYS_FL)
        # check for number of channels
        if "channel count" in ds.config["fluorescence"]:
            chc1 = ds.config["fluorescence"]["channel count"]
            chc2 = 0
            for ii in range(1, 4):
                chn = "channel {} name".format(ii)
                ecn = "fl{}_max".format(ii)
                if (chn in ds.config["fluorescence"] and
                        ecn in ds._events):
                    chc2 += 1
            if chc1 != chc2:
                msg = "Metadata: fluorescence channel count inconsistent"
                viol.append(msg)
        # check for number of lasers
        if "laser count" in ds.config["fluorescence"]:
            lsc1 = ds.config["fluorescence"]["laser count"]
            lsc2 = 0
            for ii in range(1, 4):
                kl = "laser {} lambda".format(ii)
                kp = "laser {} power".format(ii)
                if (kl in ds.config["fluorescence"] and
                        kp in ds.config["fluorescence"]):
                    lsc2 += 1
            if lsc1 != lsc2:
                msg = "Metadata: fluorescence laser count inconsistent"
                viol.append(msg)
        # check for samples per event
        if "samples per event" in ds.config["fluorescence"]:
            spe = ds.config["fluorescence"]["samples per event"]
            for key in ds["trace"].keys():
                spek = ds["trace"][key][0].size
                if spek != spe:
                    msg = "Metadata: wrong number of samples per event: " \
                          + "{} (expected {}, got {}".format(key, spe, spek)
                    viol.append(msg)
    else:
        info.append("Fluorescence: False")
    # search for missing keys (hard)
    for sec in tocheck:
        if sec not in ds.config:
            viol.append("Metadata: Missing section '{}'".format(sec))
        else:
            for key in dfn.config_keys[sec]:
                if (key in tocheck[sec] and
                        key not in ds.config[sec]):
                    viol.append("Metadata: Missing key [{}] '{}'".format(sec,
                                                                         key))
                elif (sec in tocheck_sec_aler and
                        key not in ds.config[sec]):
                    # Note: fluorescence is not treated here. It can be
                    # incomplete (e.g. number of channels installed may vary)
                    aler.append("Metadata: Missing key [{}] '{}'".format(sec,
                                                                         key))
    # search again (soft)
    for sec in tocheck_sec_aler:
        if sec in tocheck:
            # already treated above (hard)
            continue
        if sec not in ds.config:
            aler.append("Metadata: Missing section '{}'".format(sec))
        else:
            for key in dfn.config_keys[sec]:
                if key not in ds.config[sec]:
                    aler.append("Metadata: Missing key [{}] '{}'".format(sec,
                                                                         key))
    # check for medium
    if "medium" in ds.config["setup"]:
        med = ds.config["setup"]["medium"]
        if med not in ["CellCarrier", "CellCarrierB", "water", "other"]:
            msg = "Metadata: Invalid value [setup] medium: '{}'".format(med)
            viol.append(msg)
    # check for feature column names
    for feat in ds._events.keys():
        if feat not in dfn.feature_names:
            viol.append("Features: Unknown key '{}'".format(feat))
    info.append("Data file format: {}".format(ds.format))
    # hdf5-based checks
    if ds.format == "hdf5":
        # check meta data of images
        if "image" in ds._events:
            imdat = ds["image"]
            for key, val in [['CLASS', b'IMAGE'],
                             ['IMAGE_VERSION', b'1.2'],
                             ['IMAGE_SUBCLASS', b'IMAGE_GRAYSCALE']]:
                if key not in imdat.attrs:
                    aler.append("HDF5: '/image': missing attribute "
                                + "'{}'".format(key))
                elif not isinstance(imdat.attrs[key], bytes):
                    aler.append("HDF5: '/image': attribute '{}' ".format(key)
                                + "should be fixed-length ASCII string")
                elif imdat.attrs[key] != val:
                    aler.append("HDF5: '/image': attribute '{}' ".format(key)
                                + "should have value '{}'".format(val))
        # check length of logs
        with h5py.File(ds.path, mode="r") as h5:
            logs = h5["logs"]
            for logname in logs.keys():
                log = logs[logname]
                for ii in range(len(log)):
                    if len(log[ii]) > LOG_MAX_LINE_LENGTH:
                        aler.append("Logs: {} line {} ".format(logname, ii)
                                    + "exceeds maximum line length "
                                    + "{}".format(LOG_MAX_LINE_LENGTH))
    return sorted(viol), sorted(aler), sorted(info)

Response: Checks whether a dataset is complete and returns a list of str or list of str


Instruction:
Explain what the following Python 3 code does
def new_dataset(data, identifier=None):
    if isinstance(data, dict):
        return fmt_dict.RTDC_Dict(data, identifier=identifier)
    elif isinstance(data, (str_types)) or isinstance(data, pathlib.Path):
        return load_file(data, identifier=identifier)
    elif isinstance(data, RTDCBase):
        return fmt_hierarchy.RTDC_Hierarchy(data, identifier=identifier)
    else:
        msg = "data type not supported: {}".format(data.__class__)
        raise NotImplementedError(msg)

Response: Initialize a new RT - DC dataset.


Instruction:
Explain what the following Python 3 code does
def parse_log(log, url_pattern):
    hits = {}
    initial_timestamp = None

    def clean_timestamp(v):
        return ' '.join(v).strip('[]')
    for line in log:
        data = line.split()
        if not initial_timestamp:
            initial_timestamp = clean_timestamp(data[3:5])
        match = url_pattern.match(data[6])
        if match:
            ident_hash = '@'.join(match.groups())
            if ident_hash:
                hits[ident_hash] = hits.get(ident_hash, 0) + 1
    else:
        end_timestamp = clean_timestamp(data[3:5])
    return hits, initial_timestamp, end_timestamp

Response: Parse a log buffer based on url_pattern.


Instruction:
Can you create a Python 3 function that
counts the hits from a logfile.

Response:
def main(argv=None):
    """Count the hits from logfile."""
    parser = create_parser('hits_counter', description=__doc__)
    parser.add_argument('--hostname', default='cnx.org',
                        help="hostname of the site (default: cnx.org)")
    parser.add_argument('--log-format',
                        default=LOG_FORMAT_GZ, choices=LOG_FORMATS,
                        help="(default: {})".format(LOG_FORMAT_GZ))
    parser.add_argument('log_file',
                        help="path to the logfile.")
    args = parser.parse_args(argv)

    opener = LOG_FORMAT_OPENERS_MAPPING[args.log_format]

    # Build the URL pattern.
    hostname = args.hostname.replace('.', '\.')
    url_pattern = URL_PATTERN_TMPLT.format(hostname)
    url_pattern = re.compile(url_pattern)

    # Parse the log to structured data.
    with opener(args.log_file) as log:
        hits, start_timestamp, end_timestamp = parse_log(log, url_pattern)

    # Parse the configuration file for the postgres connection string.
    settings = get_app_settings_from_arguments(args)

    # Insert the hits into the database.
    connection_string = settings[config.CONNECTION_STRING]
    db_connection = psycopg2.connect(connection_string)
    with db_connection:
        with db_connection.cursor() as cursor:
            for ident_hash, hit_count in hits.items():
                cursor.execute("""\
                      INSERT INTO document_hits
                        (documentid, start_timestamp, end_timestamp, hits)
                      SELECT module_ident, %s, %s, %s
                      FROM modules WHERE
                        ident_hash(uuid, major_version, minor_version) = %s""",
                               (start_timestamp, end_timestamp, hit_count,
                                ident_hash))
            cursor.execute("SELECT update_hit_ranks();")
    db_connection.close()
    return 0


Instruction:
Can you implement a function in Python 3 that
converts a string to a boolean

Response:
def fbool(value):
    """boolean"""
    if isinstance(value, str_types):
        value = value.lower()
        if value == "false":
            value = False
        elif value == "true":
            value = True
        elif value:
            value = bool(float(value))
        else:
            raise ValueError("empty string")
    else:
        value = bool(float(value))
    return value


Instruction:
Here you have a function in Python 3, explain what it does
def fint(value):
    if isinstance(value, str_types):
        # strings might have been saved wrongly as booleans
        value = value.lower()
        if value == "false":
            value = 0
        elif value == "true":
            value = 1
        elif value:
            value = int(float(value))
        else:
            raise ValueError("empty string")
    else:
        value = int(float(value))
    return value

Response: converts a string to an integer


Instruction:
Here you have a function in Python 3, explain what it does
def fintlist(alist):
    outlist = []
    if not isinstance(alist, (list, tuple)):
        # we have a string (comma-separated integers)
        alist = alist.strip().strip("[] ").split(",")
    for it in alist:
        if it:
            outlist.append(fint(it))
    return outlist

Response: A list of integers


Instruction:
Can you write a function in Python 3 where it
exports filtered event images to an avi file.

Response:
def avi(self, path, filtered=True, override=False):
        """Exports filtered event images to an avi file

        Parameters
        ----------
        path: str
            Path to a .tsv file. The ending .tsv is added automatically.
        filtered: bool
            If set to `True`, only the filtered data (index in ds._filter)
            are used.
        override: bool
            If set to `True`, an existing file ``path`` will be overridden.
            If set to `False`, raises `OSError` if ``path`` exists.

        Notes
        -----
        Raises OSError if current dataset does not contain image data
        """
        path = pathlib.Path(path)
        ds = self.rtdc_ds
        # Make sure that path ends with .avi
        if path.suffix != ".avi":
            path = path.with_name(path.name + ".avi")
        # Check if file already exist
        if not override and path.exists():
            raise OSError("File already exists: {}\n".format(
                str(path).encode("ascii", "ignore")) +
                "Please use the `override=True` option.")
        # Start exporting
        if "image" in ds:
            # Open video for writing
            vout = imageio.get_writer(uri=path,
                                      format="FFMPEG",
                                      fps=25,
                                      codec="rawvideo",
                                      pixelformat="yuv420p",
                                      macro_block_size=None,
                                      ffmpeg_log_level="error")
            # write the filtered frames to avi file
            for evid in np.arange(len(ds)):
                # skip frames that were filtered out
                if filtered and not ds._filter[evid]:
                    continue
                try:
                    image = ds["image"][evid]
                except BaseException:
                    warnings.warn("Could not read image {}!".format(evid),
                                  NoImageWarning)
                    continue
                else:
                    if np.isnan(image[0, 0]):
                        # This is a nan-valued image
                        image = np.zeros_like(image, dtype=np.uint8)
                # Convert image to RGB
                image = image.reshape(image.shape[0], image.shape[1], 1)
                image = np.repeat(image, 3, axis=2)
                vout.append_data(image)
        else:
            msg = "No image data to export: dataset {} !".format(ds.title)
            raise OSError(msg)


Instruction:
Can you create a Python 3 function that
exports the data of an RT - DC dataset to a. fcs file.

Response:
def fcs(self, path, features, filtered=True, override=False):
        """Export the data of an RT-DC dataset to an .fcs file

        Parameters
        ----------
        mm: instance of dclab.RTDCBase
            The dataset that will be exported.
        path: str
            Path to a .tsv file. The ending .tsv is added automatically.
        features: list of str
            The features in the resulting .tsv file. These are strings
            that are defined in `dclab.definitions.scalar_feature_names`,
            e.g. "area_cvx", "deform", "frame", "fl1_max", "aspect".
        filtered: bool
            If set to `True`, only the filtered data (index in ds._filter)
            are used.
        override: bool
            If set to `True`, an existing file ``path`` will be overridden.
            If set to `False`, raises `OSError` if ``path`` exists.

        Notes
        -----
        Due to incompatibility with the .fcs file format, all events with
        NaN-valued features are not exported.
        """
        features = [c.lower() for c in features]
        ds = self.rtdc_ds

        path = pathlib.Path(path)
        # Make sure that path ends with .fcs
        if path.suffix != ".fcs":
            path = path.with_name(path.name + ".fcs")
        # Check if file already exist
        if not override and path.exists():
            raise OSError("File already exists: {}\n".format(
                str(path).encode("ascii", "ignore")) +
                "Please use the `override=True` option.")
        # Check that features are in dfn.scalar_feature_names
        for c in features:
            if c not in dfn.scalar_feature_names:
                msg = "Unknown or unsupported feature name: {}".format(c)
                raise ValueError(msg)

        # Collect the header
        chn_names = [dfn.feature_name2label[c] for c in features]

        # Collect the data
        if filtered:
            data = [ds[c][ds._filter] for c in features]
        else:
            data = [ds[c] for c in features]

        data = np.array(data).transpose()
        fcswrite.write_fcs(filename=str(path),
                           chn_names=chn_names,
                           data=data)


Instruction:
Can you tell what is the following Python 3 function doing
def hdf5(self, path, features, filtered=True, override=False,
             compression="gzip"):
        """Export the data of the current instance to an HDF5 file

        Parameters
        ----------
        path: str
            Path to an .rtdc file. The ending .rtdc is added
            automatically.
        features: list of str
            The features in the resulting .tsv file. These are strings
            that are defined in `dclab.definitions.feature_names`, e.g.
            "area_cvx", "deform", "frame", "fl1_max", "image".
        filtered: bool
            If set to `True`, only the filtered data (index in ds._filter)
            are used.
        override: bool
            If set to `True`, an existing file ``path`` will be overridden.
            If set to `False`, raises `OSError` if ``path`` exists.
        compression: str or None
            Compression method for "contour", "image", and "trace" data
            as well as logs; one of [None, "lzf", "gzip", "szip"].
        """
        path = pathlib.Path(path)
        # Make sure that path ends with .rtdc
        if not path.suffix == ".rtdc":
            path = path.parent / (path.name + ".rtdc")
        # Check if file already exist
        if not override and path.exists():
            raise OSError("File already exists: {}\n".format(path)
                          + "Please use the `override=True` option.")
        elif path.exists():
            path.unlink()

        meta = {}

        # only export configuration meta data (no user-defined config)
        for sec in dfn.CFG_METADATA:
            if sec in ["fmt_tdms"]:
                # ignored sections
                continue
            if sec in self.rtdc_ds.config:
                meta[sec] = self.rtdc_ds.config[sec].copy()

        if filtered:
            filtarr = self.rtdc_ds.filter.all
            nev = np.sum(filtarr)
        else:
            nev = len(self.rtdc_ds)
            filtarr = np.ones(nev, dtype=bool)

        # update number of events
        meta["experiment"]["event count"] = nev

        # write meta data
        with write(path_or_h5file=path, meta=meta, mode="append") as h5obj:
            # write each feature individually
            for feat in features:
                # event-wise, because
                # - tdms-based datasets don't allow indexing with numpy
                # - there might be memory issues
                if feat == "contour":
                    cont_list = []
                    cmax = 0
                    for ii in range(len(self.rtdc_ds)):
                        if filtarr[ii]:
                            dat = self.rtdc_ds["contour"][ii]
                            cont_list.append(dat)
                            cmax = max(cmax, dat.max())
                    write(h5obj,
                          data={"contour": cont_list},
                          mode="append",
                          compression=compression)
                elif feat in ["mask", "image"]:
                    # store image stacks (reduced file size and save time)
                    m = 64
                    im0 = self.rtdc_ds[feat][0]
                    imstack = np.zeros((m, im0.shape[0], im0.shape[1]),
                                       dtype=im0.dtype)
                    jj = 0
                    for ii in range(len(self.rtdc_ds)):
                        if filtarr[ii]:
                            dat = self.rtdc_ds[feat][ii]
                            imstack[jj] = dat
                            if (jj + 1) % m == 0:
                                jj = 0
                                write(h5obj,
                                      data={feat: imstack},
                                      mode="append",
                                      compression=compression)
                            else:
                                jj += 1
                    # write rest
                    if jj:
                        write(h5obj,
                              data={feat: imstack[:jj, :, :]},
                              mode="append",
                              compression=compression)
                elif feat == "trace":
                    for tr in self.rtdc_ds["trace"].keys():
                        tr0 = self.rtdc_ds["trace"][tr][0]
                        trdat = np.zeros((nev, tr0.size), dtype=tr0.dtype)
                        jj = 0
                        for ii in range(len(self.rtdc_ds)):
                            if filtarr[ii]:
                                trdat[jj] = self.rtdc_ds["trace"][tr][ii]
                                jj += 1
                        write(h5obj,
                              data={"trace": {tr: trdat}},
                              mode="append",
                              compression=compression)
                elif feat == "index" and filtered:
                    # re-enumerate data index feature (filtered data)
                    write(h5obj,
                          data={"index": np.arange(1, nev+1)},
                          mode="append")
                else:
                    write(h5obj,
                          data={feat: self.rtdc_ds[feat][filtarr]},
                          mode="append")

Response: Export the current instance data to an HDF5 file.


Instruction:
Given the following Python 3 function, write the documentation
def tsv(self, path, features, filtered=True, override=False):
        features = [c.lower() for c in features]
        path = pathlib.Path(path)
        ds = self.rtdc_ds
        # Make sure that path ends with .tsv
        if path.suffix != ".tsv":
            path = path.with_name(path.name + ".tsv")
        # Check if file already exist
        if not override and path.exists():
            raise OSError("File already exists: {}\n".format(
                str(path).encode("ascii", "ignore")) +
                "Please use the `override=True` option.")
        # Check that features are in dfn.scalar_feature_names
        for c in features:
            if c not in dfn.scalar_feature_names:
                raise ValueError("Unknown feature name {}".format(c))

        # Open file
        with path.open("w") as fd:
            # write header
            header1 = "\t".join([c for c in features])
            fd.write("# "+header1+"\n")
            header2 = "\t".join([dfn.feature_name2label[c] for c in features])
            fd.write("# "+header2+"\n")

        with path.open("ab") as fd:
            # write data
            if filtered:
                data = [ds[c][ds._filter] for c in features]
            else:
                data = [ds[c] for c in features]

            np.savetxt(fd,
                       np.array(data).transpose(),
                       fmt=str("%.10e"),
                       delimiter="\t")

Response: Export the data of the current instance to a. tsv file.


Instruction:
Create a Python 3 function for
computing md5 hex - hash of a file

Response:
def hashfile(fname, blocksize=65536, count=0):
    """Compute md5 hex-hash of a file

    Parameters
    ----------
    fname: str
        path to the file
    blocksize: int
        block size in bytes read from the file
        (set to `0` to hash the entire file)
    count: int
        number of blocks read from the file
    """
    hasher = hashlib.md5()
    fname = pathlib.Path(fname)
    with fname.open('rb') as fd:
        buf = fd.read(blocksize)
        ii = 0
        while len(buf) > 0:
            hasher.update(buf)
            buf = fd.read(blocksize)
            ii += 1
            if count and ii == count:
                break
    return hasher.hexdigest()


Instruction:
Can you implement a function in Python 3 that
converts an object to a string representation for hashing.

Response:
def obj2str(obj):
    """String representation of an object for hashing"""
    if isinstance(obj, str_types):
        return obj.encode("utf-8")
    elif isinstance(obj, pathlib.Path):
        return obj2str(str(obj))
    elif isinstance(obj, (bool, int, float)):
        return str(obj).encode("utf-8")
    elif obj is None:
        return b"none"
    elif isinstance(obj, np.ndarray):
        return obj.tostring()
    elif isinstance(obj, tuple):
        return obj2str(list(obj))
    elif isinstance(obj, list):
        return b"".join(obj2str(o) for o in obj)
    elif isinstance(obj, dict):
        return obj2str(list(obj.items()))
    elif hasattr(obj, "identifier"):
        return obj2str(obj.identifier)
    elif isinstance(obj, h5py.Dataset):
        return obj2str(obj[0])
    else:
        raise ValueError("No rule to convert object '{}' to string.".
                         format(obj.__class__))


Instruction:
Create a Python 3 function for
creating a self - signed certificate key pair.

Response:
def create_self_signed_cert():
    """Creates a self-signed certificate key pair."""
    config_dir = Path(BaseDirectory.xdg_config_home) / 'xenon-grpc'
    config_dir.mkdir(parents=True, exist_ok=True)

    key_prefix = gethostname()
    crt_file = config_dir / ('%s.crt' % key_prefix)
    key_file = config_dir / ('%s.key' % key_prefix)

    if crt_file.exists() and key_file.exists():
        return crt_file, key_file

    logger = logging.getLogger('xenon')
    logger.info("Creating authentication keys for xenon-grpc.")

    # create a key pair
    k = crypto.PKey()
    k.generate_key(crypto.TYPE_RSA, 1024)

    # create a self-signed cert
    cert = crypto.X509()
    cert.get_subject().CN = gethostname()
    cert.set_serial_number(1000)
    cert.gmtime_adj_notBefore(0)
    # valid for almost ten years!
    cert.gmtime_adj_notAfter(10 * 365 * 24 * 3600)
    cert.set_issuer(cert.get_subject())
    cert.set_pubkey(k)
    cert.sign(k, 'sha256')

    open(str(crt_file), "wb").write(
        crypto.dump_certificate(crypto.FILETYPE_PEM, cert))
    open(str(key_file), "wb").write(
        crypto.dump_privatekey(crypto.FILETYPE_PEM, k))

    return crt_file, key_file


Instruction:
Create a Python 3 function to
get the project name from a path.

Response:
def get_project_name_from_path(path, append_mx=False):
    """Get the project name from a path.

    For a path "/home/peter/hans/HLC12398/online/M1_13.tdms" or
    For a path "/home/peter/hans/HLC12398/online/data/M1_13.tdms" or
    without the ".tdms" file, this will return always "HLC12398".

    Parameters
    ----------
    path: str
        path to tdms file
    append_mx: bool
        append measurement number, e.g. "M1"
    """
    path = pathlib.Path(path)
    if path.suffix == ".tdms":
        dirn = path.parent
        mx = path.name.split("_")[0]
    elif path.is_dir():
        dirn = path
        mx = ""
    else:
        dirn = path.parent
        mx = ""

    project = ""
    if mx:
        # check para.ini
        para = dirn / (mx + "_para.ini")
        if para.exists():
            with para.open() as fd:
                lines = fd.readlines()
            for line in lines:
                if line.startswith("Sample Name ="):
                    project = line.split("=")[1].strip()
                    break

    if not project:
        # check if the directory contains data or is online
        root1, trail1 = dirn.parent, dirn.name
        root2, trail2 = root1.parent, root1.name
        trail3 = root2.name

        if trail1.lower() in ["online", "offline"]:
            # /home/peter/hans/HLC12398/online/
            project = trail2
        elif (trail1.lower() == "data" and
              trail2.lower() in ["online", "offline"]):
            # this is olis new folder sctructure
            # /home/peter/hans/HLC12398/online/data/
            project = trail3
        else:
            project = trail1

    if append_mx:
        project += " - " + mx

    return project


Instruction:
Make a summary of the following Python 3 code
def get_tdms_files(directory):
    path = pathlib.Path(directory).resolve()
    # get all tdms files
    tdmslist = [r for r in path.rglob("*.tdms") if r.is_file()]
    # exclude traces files
    tdmslist = [r for r in tdmslist if not r.name.endswith("_traces.tdms")]
    return sorted(tdmslist)

Response:
Recursively find projects based on. tdms file endings
    Searches the directory and returns a sorted list
    of all found. tdms project files except fluorescence
    data trace files which end with _traces. tdms.


Instruction:
Make a summary of the following Python 3 code
def _init_data_with_tdms(self, tdms_filename):
        tdms_file = TdmsFile(str(tdms_filename))
        # time is always there
        table = "Cell Track"
        # Edit naming.dclab2tdms to add features
        for arg in naming.tdms2dclab:
            try:
                data = tdms_file.object(table, arg).data
            except KeyError:
                pass
            else:
                if data is None or len(data) == 0:
                    # Ignore empty features. npTDMS treats empty
                    # features in the following way:
                    # - in nptdms 0.8.2, `data` is `None`
                    # - in nptdms 0.9.0, `data` is an array of length 0
                    continue
                self._events[naming.tdms2dclab[arg]] = data

        # Set up configuration
        tdms_config = Configuration(
            files=[self.path.with_name(self._mid + "_para.ini"),
                   self.path.with_name(self._mid + "_camera.ini")],
        )
        dclab_config = Configuration()
        for section in naming.configmap:
            for pname in naming.configmap[section]:
                meta = naming.configmap[section][pname]
                typ = dfn.config_funcs[section][pname]
                if isinstance(meta, tuple):
                    osec, opar = meta
                    if osec in tdms_config and opar in tdms_config[osec]:
                        val = tdms_config[osec].pop(opar)
                        dclab_config[section][pname] = typ(val)
                else:
                    dclab_config[section][pname] = typ(meta)

        self.config = dclab_config
        self._complete_config_tdms(tdms_config)

        self._init_filters()

Response: Initializes the current RT - DC dataset with a tdms file.


Instruction:
Here you have a function in Python 3, explain what it does
def hash(self):
        if self._hash is None:
            # Only hash _camera.ini and _para.ini
            fsh = [self.path.with_name(self._mid + "_camera.ini"),
                   self.path.with_name(self._mid + "_para.ini")]
            tohash = [hashfile(f) for f in fsh]
            tohash.append(self.path.name)
            # Hash a maximum of ~1MB of the tdms file
            tohash.append(hashfile(self.path, blocksize=65536, count=20))
            self._hash = hashobj(tohash)
        return self._hash

Response: Hash value based on file name and. ini file content


Instruction:
Given the following Python 3 function, write the documentation
def determine_offset(self):
        # In case of regular RTDC, the first contour is
        # missing. In case of fRTDC, it is there, so we
        # might have an offset. We find out if the first
        # contour frame is missing by comparing it to
        # the "frame" column of the rtdc dataset.
        fref = self._contour_data.get_frame(0)
        f0 = self.frame[0]
        f1 = self.frame[1]
        # Use allclose to avoid float/integer comparison problems
        if np.allclose(fref, f0):
            self.event_offset = 0
        elif np.allclose(fref, f1):
            self.event_offset = 1
        else:
            msg = "Contour data has unknown offset (frame {})!".format(fref)
            raise IndexError(msg)
        self._initialized = True

Response: Determines the offset of the contours w. r. t. other data columns.


Instruction:
Write a Python 3 function for
trying to find a contour file that belongs to an RTDC dataset. Returns None if no contour file is found.

Response:
def find_contour_file(rtdc_dataset):
        """Tries to find a contour file that belongs to an RTDC dataset

        Returns None if no contour file is found.
        """
        cont_id = rtdc_dataset.path.stem
        cands = [c.name for c in rtdc_dataset._fdir.rglob("*_contours.txt")]
        cands = sorted(cands)
        # Search for perfect matches, e.g.
        # - M1_0.240000ul_s.tdms
        # - M1_0.240000ul_s_contours.txt
        for c1 in cands:
            if c1.startswith(cont_id):
                cfile = rtdc_dataset._fdir / c1
                break
        else:
            # Search for M* matches with most overlap, e.g.
            # - M1_0.240000ul_s.tdms
            # - M1_contours.txt
            for c2 in cands:
                if (c2.split("_")[0] == rtdc_dataset._mid):
                    # Do not confuse with M10_contours.txt
                    cfile = rtdc_dataset._fdir / c2
                    break
            else:
                msg = "No contour data found for {}".format(rtdc_dataset)
                warnings.warn(msg, NoContourDataWarning)
                cfile = None
        return cfile


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _index_file(self):
        with self.filename.open() as fd:
            data = fd.read()

        ident = "Contour in frame"
        self._data = data.split(ident)[1:]
        self._initialized = True

Response: Open and index the contour file and populate internal list of contours with the names of the contour entries.


Instruction:
Can you create a Python 3 function that
returns the frame number of a contour

Response:
def get_frame(self, idx):
        """Return the frame number of a contour"""
        cont = self.data[idx]
        frame = int(cont.strip().split(" ", 1)[0])
        return frame


Instruction:
Implement a function in Python 3 to
return the viscosity for the given medium and channel width.

Response:
def get_viscosity(medium="CellCarrier", channel_width=20.0, flow_rate=0.16,
                  temperature=23.0):
    """Returns the viscosity for RT-DC-specific media

    Parameters
    ----------
    medium: str
        The medium to compute the viscosity for.
        One of ["CellCarrier", "CellCarrier B", "water"].
    channel_width: float
        The channel width in µm
    flow_rate: float
        Flow rate in µl/s
    temperature: float or ndarray
        Temperature in °C

    Returns
    -------
    viscosity: float or ndarray
        Viscosity in mPa*s

    Notes
    -----
    - CellCarrier and CellCarrier B media are optimized for
      RT-DC measurements.
    - Values for the viscosity of water are computed using
      equation (15) from :cite:`Kestin_1978`.
    """
    if medium.lower() not in ["cellcarrier", "cellcarrier b", "water"]:
        raise ValueError("Invalid medium: {}".format(medium))

    # convert flow_rate from µl/s to m³/s
    # convert channel_width from µm to m
    term1 = 1.1856 * 6 * flow_rate * 1e-9 / (channel_width * 1e-6)**3 * 2 / 3

    if medium == "CellCarrier":
        temp_corr = (temperature / 23.2)**-0.866
        term2 = 0.6771 / 0.5928 + 0.2121 / (0.5928 * 0.677)
        eta = 0.179 * (term1 * term2)**(0.677 - 1) * temp_corr * 1e3
    elif medium == "CellCarrier B":
        temp_corr = (temperature / 23.6)**-0.866
        term2 = 0.6771 / 0.5928 + 0.2121 / (0.5928 * 0.634)
        eta = 0.360 * (term1 * term2)**(0.634 - 1) * temp_corr * 1e3
    elif medium == "water":
        # see equation (15) in Kestin et al, J. Phys. Chem. 7(3) 1978
        if np.min(temperature) < 0 or np.max(temperature) > 40:
            msg = "For water, the temperature must be in [0, 40] degC! " \
                  "Got min/max values of '{}'.".format(np.min(temperature),
                                                       np.max(temperature))
            raise ValueError(msg)
        eta0 = 1.002  # [mPa]
        right = (20-temperature) / (temperature + 96) \
            * (+ 1.2364
               - 1.37e-3 * (20 - temperature)
               + 5.7e-6 * (20 - temperature)**2
               )
        eta = eta0 * 10**right
    return eta


Instruction:
Write a Python 3 script for
computing statistics for an RT - DC dataset.

Response:
def get_statistics(ds, methods=None, features=None):
    """Compute statistics for an RT-DC dataset

    Parameters
    ----------
    ds: dclab.rtdc_dataset.RTDCBase
        The dataset for which to compute the statistics.
    methods: list of str or None
        The methods wih which to compute the statistics.
        The list of available methods is given with
        `dclab.statistics.Statistics.available_methods.keys()`
        If set to `None`, statistics for all methods are computed.
    features: list of str
        Feature name identifiers are defined in
        `dclab.definitions.scalar_feature_names`.
        If set to `None`, statistics for all axes are computed.

    Returns
    -------
    header: list of str
        The header (feature + method names) of the computed statistics.
    values: list of float
        The computed statistics.
    """
    if methods is None:
        cls = list(Statistics.available_methods.keys())
        # sort the features in a usable way
        avm = Statistics.available_methods
        me1 = [m for m in cls if not avm[m].req_feature]
        me2 = [m for m in cls if avm[m].req_feature]
        methods = me1 + me2

    if features is None:
        features = dfn.scalar_feature_names
    else:
        features = [a.lower() for a in features]

    header = []
    values = []

    # To make sure that all methods are computed for each feature in a block,
    # we loop over all features. It would be easier to loop over the methods,
    # but the resulting statistics would not be human-friendly.
    for ft in features:
        for mt in methods:
            meth = Statistics.available_methods[mt]
            if meth.req_feature:
                if ft in ds:
                    values.append(meth(ds=ds, feature=ft))
                else:
                    values.append(np.nan)
                header.append(" ".join([mt, dfn.feature_name2label[ft]]))
            else:
                # Prevent multiple entries of this method.
                if not header.count(mt):
                    values.append(meth(ds=ds))
                    header.append(mt)

    return header, values


Instruction:
Write a Python 3 function for
computing an intelligent value for the mode of the current node in the experimental.

Response:
def mode(data):
    """Compute an intelligent value for the mode

    The most common value in experimental is not very useful if there
    are a lot of digits after the comma. This method approaches this
    issue by rounding to bin size that is determined by the
    Freedman–Diaconis rule.

    Parameters
    ----------
    data: 1d ndarray
        The data for which the mode should be computed.

    Returns
    -------
    mode: float
        The mode computed with the Freedman-Diaconis rule.
    """
    # size
    n = data.shape[0]
    # interquartile range
    iqr = np.percentile(data, 75)-np.percentile(data, 25)
    # Freedman–Diaconis
    bin_size = 2 * iqr / n**(1/3)

    if bin_size == 0:
        return np.nan

    # Add bin_size/2, because we want the center of the bin and
    # not the left corner of the bin.
    databin = np.round(data/bin_size)*bin_size + bin_size/2
    u, indices = np.unique(databin, return_inverse=True)
    mode = u[np.argmax(np.bincount(indices))]

    return mode


Instruction:
Create a Python 3 function to
return the feature data for the specified feature

Response:
def get_feature(self, ds, feat):
        """Return filtered feature data

        The features are filtered according to the user-defined filters,
        using the information in `ds._filter`. In addition, all
        `nan` and `inf` values are purged.

        Parameters
        ----------
        ds: dclab.rtdc_dataset.RTDCBase
            The dataset containing the feature
        feat: str
            The name of the feature; must be a scalar feature
        """
        if ds.config["filtering"]["enable filters"]:
            x = ds[feat][ds._filter]
        else:
            x = ds[feat]
        bad = np.isnan(x) | np.isinf(x)
        xout = x[~bad]
        return xout


Instruction:
Write a Python 3 function for
checking if the global resource set is valid.

Response:
def check_global_valid(self):
        """
        Makes sure there aren't any SAM resources in a template that will be used in a CloudFormation StackSet
        :return: bool
        """
        serverless_cnt = len(list(filter(lambda x: x._serverless_type, self._data['resources'])))
        if serverless_cnt > 0:
            return False
        return True


Instruction:
Can you implement a function in Python 3 that
checks if a CloudFormation stack with given name exists.

Response:
def has_stack(self, stack_name):
        """
        Checks if a CloudFormation stack with given name exists
        :param stack_name: Name or ID of the stack
        :return: True if stack exists. False otherwise
        """
        cf = self.cf_client
        try:
            resp = cf.describe_stacks(StackName=stack_name)
            if len(resp["Stacks"]) != 1:
                return False

            # When you run CreateChangeSet on a a stack that does not exist,
            # CloudFormation will create a stack and set it's status
            # REVIEW_IN_PROGRESS. However this stack is cannot be manipulated
            # by "update" commands. Under this circumstances, we treat like
            # this stack does not exist and call CreateChangeSet will
            # ChangeSetType set to CREATE and not UPDATE.
            stack = resp["Stacks"][0]
            return stack["StackStatus"] != "REVIEW_IN_PROGRESS"

        except botocore.exceptions.ClientError as e:
            # If a stack does not exist, describe_stacks will throw an
            # exception. Unfortunately we don't have a better way than parsing
            # the exception msg to understand the nature of this exception.
            msg = str(e)

            if "Stack with id {0} does not exist".format(stack_name) in msg:
                LOG.debug("Stack with id {0} does not exist".format(
                    stack_name))
                return False
            else:
                # We don't know anything about this exception. Don't handle
                LOG.debug("Unable to get stack details.", exc_info=e)
                raise e


Instruction:
How would you explain what the following Python 3 function does
def has_key(cls, *args):
        key = args if len(args) > 1 else args[0]
        return key in cls._instances

Response: Check whether a flyweight object with the specified key has already been created.


Instruction:
Explain what the following Python 3 code does
def from_key(cls, *args):
        key = args if len(args) > 1 else args[0]
        return cls._instances.get(key, None)

Response: Returns a new object with the given key if it has already been created.


Instruction:
Given the following Python 3 function, write the documentation
def write(self, name, **data):

        data["name"] = name
        if not ("timestamp" in data):
            data["timestamp"] = datetime.utcnow()

        try:
            self.client.index(
                index=self.get_index(),
                doc_type=self.doc_type,
                id=None,
                body=data
            )
        except TransportError as exc:
            logger.warning('writing metric %r failure %r', data, exc)

Response: Writes the metric to elasticsearch


Instruction:
How would you explain what the following Python 3 function does
def bulk_write(self, metrics):
        actions = []
        index = self.get_index()
        for metric in metrics:
            actions.append({'index': {'_index': index, '_type': self.doc_type}})
            actions.append(metric)
        try:
            self.client.bulk(actions)
        except TransportError as exc:
            logger.warning('bulk_write metrics %r failure %r', metrics, exc)

Response: Write multiple metrics to elasticsearch in one request


Instruction:
Create a Python 3 function for
computing average and standard deviation of the event brightness of the contour.

Response:
def get_bright(mask, image, ret_data="avg,sd"):
    """Compute avg and/or std of the event brightness

    The event brightness is defined by the gray-scale values of the
    image data within the event mask area.

    Parameters
    ----------
    mask: ndarray or list of ndarrays of shape (M,N) and dtype bool
        The mask values, True where the event is located in `image`.
    image: ndarray or list of ndarrays of shape (M,N)
        A 2D array that holds the image in form of grayscale values
        of an event.
    ret_data: str
        A comma-separated list of metrices to compute
        - "avg": compute the average
        - "sd": compute the standard deviation
        Selected metrics are returned in alphabetical order.

    Returns
    -------
    bright_avg: float or ndarray of size N
        Average image data within the contour
    bright_std: float or ndarray of size N
        Standard deviation of image data within the contour
    """
    # This method is based on a pull request by Maik Herbig.
    ret_avg = "avg" in ret_data
    ret_std = "sd" in ret_data

    if ret_avg + ret_std == 0:
        raise ValueError("No valid metrices selected!")

    if isinstance(mask, np.ndarray) and len(mask.shape) == 2:
        # We have a single image
        image = [image]
        mask = [mask]
        ret_list = False
    else:
        ret_list = True

    length = min(len(mask), len(image))

    # Results are stored in a separate array initialized with nans
    if ret_avg:
        avg = np.zeros(length, dtype=float) * np.nan
    if ret_std:
        std = np.zeros(length, dtype=float) * np.nan

    for ii in range(length):
        imgi = image[ii]
        mski = mask[ii]
        # Assign results
        if ret_avg:
            avg[ii] = np.mean(imgi[mski])
        if ret_std:
            std[ii] = np.std(imgi[mski])

    results = []
    # Keep alphabetical order
    if ret_avg:
        results.append(avg)
    if ret_std:
        results.append(std)

    if not ret_list:
        # Only return scalars
        results = [r[0] for r in results]

    if ret_avg+ret_std == 1:
        # Only return one column
        return results[0]

    return results


Instruction:
How would you code a function in Python 3 to
search archive contents. Look up search results in cache, if not in cache, do a database search and cache the result

Response:
def search(query, query_type, nocache=False):
    """Search archive contents.

    Look up search results in cache, if not in cache,
    do a database search and cache the result
    """
    settings = get_current_registry().settings
    memcache_servers = settings['memcache-servers'].split()
    if not memcache_servers:
        # memcache is not enabled, do a database search directly
        return database_search(query, query_type)

    # sort query params and create a key for the search
    search_params = []
    search_params += copy.deepcopy(query.terms)
    search_params += copy.deepcopy(query.filters)
    search_params += [('sort', i) for i in query.sorts]
    search_params.sort(key=lambda record: (record[0], record[1]))
    search_params.append(('query_type', query_type))

    # search_key should look something like:
    # '"sort:pubDate" "text:college physics" "query_type:weakAND"'
    search_key = u' '.join([u'"{}"'.format(u':'.join(param))
                           for param in search_params])
    # hash the search_key so it never exceeds the key length limit (250) in
    # memcache
    mc_search_key = binascii.hexlify(
        hashlib.pbkdf2_hmac('sha1', search_key.encode('utf-8'), b'', 1))

    # look for search results in memcache first, unless nocache
    mc = memcache.Client(memcache_servers,
                         server_max_value_length=128*1024*1024, debug=0)

    if not nocache:
        search_results = mc.get(mc_search_key)
    else:
        search_results = None

    if not search_results:
        # search results is not in memcache, do a database search
        search_results = database_search(query, query_type)

        cache_length = int(settings['search-cache-expiration'])

        # for particular searches, store in memcache for longer
        if (len(search_params) == 2 and
                # search by subject
                search_params[0][0] == 'subject' or
                # search single terms
                search_params[0][0] == 'text' and
                                       ' ' not in search_params[0][1]):
                # search with one term or one filter, plus query_type
            cache_length = int(settings['search-long-cache-expiration'])

        # store in memcache
        mc.set(mc_search_key, search_results, time=cache_length,
               min_compress_len=1024*1024)  # compress when > 1MB

    # return search results
    return search_results


Instruction:
Write a Python 3 script for
determining available features for an RT - DC dataset.

Response:
def available_features(rtdc_ds):
        """Determine available features for an RT-DC dataset

        Parameters
        ----------
        rtdc_ds: instance of RTDCBase
            The dataset to check availability for

        Returns
        -------
        features: dict
            Dictionary with feature names as keys and instances
            of `AncillaryFeature` as values.
        """
        cols = {}
        for inst in AncillaryFeature.features:
            if inst.is_available(rtdc_ds):
                cols[inst.feature_name] = inst
        return cols


Instruction:
Given the following Python 3 function, write the documentation
def compute(self, rtdc_ds):
        data = self.method(rtdc_ds)
        dsize = len(rtdc_ds) - len(data)

        if dsize > 0:
            msg = "Growing feature {} in {} by {} to match event number!"
            warnings.warn(msg.format(self.feature_name, rtdc_ds, abs(dsize)),
                          BadFeatureSizeWarning)
            data.resize(len(rtdc_ds), refcheck=False)
            data[-dsize:] = np.nan
        elif dsize < 0:
            msg = "Shrinking feature {} in {} by {} to match event number!"
            warnings.warn(msg.format(self.feature_name, rtdc_ds, abs(dsize)),
                          BadFeatureSizeWarning)
            data.resize(len(rtdc_ds), refcheck=False)

        if isinstance(data, np.ndarray):
            data.setflags(write=False)
        elif isinstance(data, list):
            for item in data:
                if isinstance(item, np.ndarray):
                    item.setflags(write=False)

        return data

Response: Compute the feature with self. method


Instruction:
Create a Python 3 function for
returning all instances that compute feature_name

Response:
def get_instances(feature_name):
        """Return all all instances that compute `feature_name`"""
        feats = []
        for ft in AncillaryFeature.features:
            if ft.feature_name == feature_name:
                feats.append(ft)
        return feats


Instruction:
Implement a Python 3 function for
using for identifying an ancillary computation

Response:
def hash(self, rtdc_ds):
        """Used for identifying an ancillary computation

        The data columns and the used configuration keys/values
        are hashed.
        """
        hasher = hashlib.md5()
        # data columns
        for col in self.req_features:
            hasher.update(obj2str(rtdc_ds[col]))
        # config keys
        for sec, keys in self.req_config:
            for key in keys:
                val = rtdc_ds.config[sec][key]
                data = "{}:{}={}".format(sec, key, val)
                hasher.update(obj2str(data))
        return hasher.hexdigest()


Instruction:
Given the following Python 3 function, write the documentation
def is_available(self, rtdc_ds, verbose=False):
        # Check config keys
        for item in self.req_config:
            section, keys = item
            if section not in rtdc_ds.config:
                if verbose:
                    print("{} not in config".format(section))
                return False
            else:
                for key in keys:
                    if key not in rtdc_ds.config[section]:
                        if verbose:
                            print("{} not in config['{}']".format(key,
                                                                  section))
                        return False
        # Check features
        for col in self.req_features:
            if col not in rtdc_ds:
                return False
        # Check priorities of other features
        for of in AncillaryFeature.features:
            if of == self:
                # nothing to compare
                continue
            elif of.feature_name == self.feature_name:
                # same feature name
                if of.priority <= self.priority:
                    # lower priority, ignore
                    continue
                else:
                    # higher priority
                    if of.is_available(rtdc_ds):
                        # higher priority is available, thus
                        # this feature is not available
                        return False
                    else:
                        # higher priority not available
                        continue
            else:
                # other feature
                continue
        return True

Response: Checks whether the feature is available for the current instance of AncillaryFeature.


Instruction:
Write a Python 3 script to
return a KafkaProducer instance for this connection.

Response:
def producer(self):
        """
        :raises: kafka.errors.NoBrokersAvailable if the connection is broken
        """
        if self._producer:
            return self._producer

        self._producer = KafkaProducer(
            bootstrap_servers=self.hosts,
            value_serializer=lambda v: self._serializer_class().dumps(v).encode('utf-8'),
            **self._kwargs
        )

        return self._producer


Instruction:
Make a summary of the following Python 3 code
def write(self, name, **data):

        data["name"] = name
        if not ("timestamp" in data):
            data["timestamp"] = datetime.utcnow()

        try:
            self.producer.send(topic=self.topic, value=data)
            self.producer.flush()
        except (KafkaTimeoutError, NoBrokersAvailable) as exc:
            logger.warning('writing metric %r failure %r', data, exc)

Response: Write the metric to kafka


Instruction:
Create a Python 3 function to
write multiple metrics to kafka in one request

Response:
def bulk_write(self, metrics):
        """
        Write multiple metrics to kafka in one request

        Args:
            metrics (list):
        """
        try:
            for metric in metrics:
                self.producer.send(self.topic, metric)
            self.producer.flush()
        except (KafkaTimeoutError, NoBrokersAvailable) as exc:
            logger.warning('bulk_write metrics %r failure %r', metrics, exc)


Instruction:
How would you explain what the following Python 3 function does
def create_module_file(package, module, opts):
    if not opts.noheadings:
        text = format_heading(1, '%s module' % module)
    else:
        text = ''
    # text += format_heading(2, ':mod:`%s` Module' % module)
    text += format_directive(module, package)
    write_file(makename(package, module), text, opts)

Response: Build the text of the file and write the file.


Instruction:
Explain what the following Python 3 code does
def create_package_file(root, master_package, subroot, py_files, opts, subs):
    text = format_heading(1, '%s' % makename(master_package, subroot))

    if opts.modulefirst:
        text += format_directive(subroot, master_package)
        text += '\n'

    # build a list of directories that are szvpackages (contain an INITPY file)
    subs = [sub for sub in subs if path.isfile(path.join(root, sub, INITPY))]
    # if there are some package directories, add a TOC for theses subpackages
    if subs:
        # text += format_heading(2, 'Subpackages')
        text += '.. toctree::\n\n'
        for sub in subs:
            text += '    %s.%s\n' % (makename(master_package, subroot), sub)
        text += '\n'

    submods = [path.splitext(sub)[0] for sub in py_files
               if not shall_skip(path.join(root, sub), opts) and
               sub != INITPY]
    if submods:
        #text += format_heading(2, 'Submodules')
        if opts.separatemodules:
            text += '.. toctree::\n\n'
            for submod in submods:
                modfile = makename(master_package, makename(subroot, submod))
                text += '   %s\n' % modfile

                # generate separate file for this module
                if not opts.noheadings:
                    filetext = format_heading(1, '%s module' % modfile)
                else:
                    filetext = ''
                filetext += format_directive(makename(subroot, submod),
                                             master_package)
                write_file(modfile, filetext, opts)
        else:
            for submod in submods:
                modfile = makename(master_package, makename(subroot, submod))
                if not opts.noheadings:
                    text += format_heading(2, '%s module' % modfile)
                text += format_directive(makename(subroot, submod),
                                         master_package)
                text += '\n'
        text += '\n'

    if not opts.modulefirst:
        text += format_heading(2, 'Module contents')
        text += format_directive(subroot, master_package)

    write_file(makename(master_package, subroot), text, opts)

Response: Create the text of the file and write it.


Instruction:
Given the following Python 3 function, write the documentation
def shall_skip(module, opts):
    # skip it if there is nothing (or just \n or \r\n) in the file
    if path.getsize(module) <= 2:
        return True
    # skip if it has a "private" name and this is selected
    filename = path.basename(module)
    if filename != '__init__.py' and filename.startswith('_') and \
       not opts.includeprivate:
        return True
    return False

Response: Check if we want to skip this module.


Instruction:
How would you explain what the following Python 3 function does
def safe_stat(path, timeout=1, cmd=None):
    "Use threads and a subproc to bodge a timeout on top of filesystem access"
    global safe_stat_process

    if cmd is None:
        cmd = ['/usr/bin/stat']

    cmd.append(path)

    def target():
        global safe_stat_process
        logger.debug('Stat thread started')
        safe_stat_process = subprocess.Popen(cmd, stdout=PIPE, stderr=PIPE)
        _results = safe_stat_process.communicate()  # noqa
        logger.debug('Stat thread finished')

    thread = threading.Thread(target=target)
    thread.start()

    thread.join(timeout)

    if thread.is_alive():  # stat took longer than timeout
        safe_stat_process.terminate()
        thread.join()
    return safe_stat_process.returncode == 0

Response: Use threads and a subproc to bodge a timeout on top of filesystem access


Instruction:
Implement a function in Python 3 to
get the names of all polygon filters in the order of creation

Response:
def get_polygon_filter_names():
    """Get the names of all polygon filters in the order of creation"""
    names = []
    for p in PolygonFilter.instances:
        names.append(p.name)
    return names


Instruction:
How would you code a function in Python 3 to
check if the data given is valid

Response:
def _check_data(self):
        """Check if the data given is valid"""
        if self.axes is None:
            raise PolygonFilterError("`axes` parm not set.")
        if self.points is None:
            raise PolygonFilterError("`points` parm not set.")
        self.points = np.array(self.points)
        if self.points.shape[1] != 2:
            raise PolygonFilterError("data points' shape[1] must be 2.")
        if self.name is None:
            self.name = "polygon filter {}".format(self.unique_id)
        if not isinstance(self.inverted, bool):
            raise PolygonFilterError("`inverted` must be boolean.")


Instruction:
Implement a Python 3 function for
loading all filters from a text file

Response:
def _load(self, filename):
        """Import all filters from a text file"""
        filename = pathlib.Path(filename)
        with filename.open() as fd:
            data = fd.readlines()

        # Get the strings that correspond to self.fileid
        bool_head = [l.strip().startswith("[") for l in data]

        int_head = np.squeeze(np.where(bool_head))
        int_head = np.atleast_1d(int_head)

        start = int_head[self.fileid]+1

        if len(int_head) > self.fileid+1:
            end = int_head[self.fileid+1]
        else:
            end = len(data)

        subdata = data[start:end]

        # separate all elements and strip them
        subdata = [[it.strip() for it in l.split("=")] for l in subdata]

        points = []

        for var, val in subdata:
            if var.lower() == "x axis":
                xaxis = val.lower()
            elif var.lower() == "y axis":
                yaxis = val.lower()
            elif var.lower() == "name":
                self.name = val
            elif var.lower() == "inverted":
                if val == "True":
                    self.inverted = True
            elif var.lower().startswith("point"):
                val = np.array(val.strip("[]").split(), dtype=float)
                points.append([int(var[5:]), val])
            else:
                raise KeyError("Unknown variable: {} = {}".
                               format(var, val))
        self.axes = (xaxis, yaxis)
        # sort points
        points.sort()
        # get only coordinates from points
        self.points = np.array([p[1] for p in points])

        # overwrite unique id
        unique_id = int(data[start-1].strip().strip("Polygon []"))
        self._set_unique_id(unique_id)


Instruction:
Can you tell what is the following Python 3 function doing
def _set_unique_id(self, unique_id):
        assert isinstance(unique_id, int), "unique_id must be an integer"

        if PolygonFilter.instace_exists(unique_id):
            newid = max(PolygonFilter._instance_counter, unique_id+1)
            msg = "PolygonFilter with unique_id '{}' exists.".format(unique_id)
            msg += " Using new unique id '{}'.".format(newid)
            warnings.warn(msg, FilterIdExistsWarning)
            unique_id = newid

        ic = max(PolygonFilter._instance_counter, unique_id+1)
        PolygonFilter._instance_counter = ic
        self.unique_id = unique_id

Response: Define a unique id for this filter


Instruction:
Here you have a function in Python 3, explain what it does
def copy(self, invert=False):
        if invert:
            inverted = not self.inverted
        else:
            inverted = self.inverted

        return PolygonFilter(axes=self.axes,
                             points=self.points,
                             name=self.name,
                             inverted=inverted)

Response: Return a copy of the current instance.


Instruction:
Make a summary of the following Python 3 code
def filter(self, datax, datay):
        f = np.ones(datax.shape, dtype=bool)
        for i, p in enumerate(zip(datax, datay)):
            f[i] = PolygonFilter.point_in_poly(p, self.points)

        if self.inverted:
            np.invert(f, f)

        return f

Response: Filter a set of datax and datay according to self. points


Instruction:
Can you implement a function in Python 3 that
gets an instance of the PolygonFilter using a unique id

Response:
def get_instance_from_id(unique_id):
        """Get an instance of the `PolygonFilter` using a unique id"""
        for instance in PolygonFilter.instances:
            if instance.unique_id == unique_id:
                return instance
        # if this does not work:
        raise KeyError("PolygonFilter with unique_id {} not found.".
                       format(unique_id))


Instruction:
Given the following Python 3 function, write the documentation
def import_all(path):
        plist = []
        fid = 0
        while True:
            try:
                p = PolygonFilter(filename=path, fileid=fid)
                plist.append(p)
                fid += 1
            except IndexError:
                break
        return plist

Response: Import all polygons from a. poly file.


Instruction:
Can you create a Python 3 function that
determines whether a point is inside a polygon area.

Response:
def point_in_poly(p, poly):
        """Determine whether a point is within a polygon area

        Uses the ray casting algorithm.

        Parameters
        ----------
        p: float
            Coordinates of the point
        poly: array_like of shape (N, 2)
            Polygon (`PolygonFilter.points`)

        Returns
        -------
        inside: bool
            `True`, if point is inside.

        Notes
        -----
        If `p` lies on a side of the polygon, it is defined as

        - "inside" if it is on the top or right
        - "outside" if it is on the lower or left
        """
        poly = np.array(poly)
        n = poly.shape[0]
        inside = False
        x, y = p

        # Coarse bounding box exclusion:
        if (x <= poly[:, 0].max() and x > poly[:, 0].min()
                and y <= poly[:, 1].max() and y > poly[:, 1].min()):
            # The point is within the coarse bounding box.
            p1x, p1y = poly[0]  # point i in contour
            for ii in range(n):  # also covers (n-1, 0) (circular)
                p2x, p2y = poly[(ii+1) % n]  # point ii+1 in contour (circular)
                # Edge-wise fine bounding-ray exclusion.
                # Determine whether point is in the current ray,
                # defined by the y-range of p1 and p2 and whether
                # it is left of p1 and p2.
                if (y > min(p1y, p2y) and y <= max(p1y, p2y)  # in y-range
                        and x <= max(p1x, p2x)):  # left of p1 and p2
                    # Note that always p1y!=p2y due to the above test.
                    # Only Compute the x-coordinate of the intersection
                    # between line p1-p2 and the horizontal ray,
                    # ((y-p1y)*(p2x-p1x)/(p2y-p1y) + p1x),
                    # if x is not already known to be left of it
                    # (p1x==p2x in combination with x<=max(p1x, p2x) above).
                    if p1x == p2x or x <= (y-p1y)*(p2x-p1x)/(p2y-p1y) + p1x:
                        # Toggle `inside` if the ray intersects
                        # with the current edge.
                        inside = not inside
                # Move on to the next edge of the polygon.
                p1x, p1y = p2x, p2y

        return inside


Instruction:
Write a Python 3 script for
removing a polygon filter from PolygonFilter. instances

Response:
def remove(unique_id):
        """Remove a polygon filter from `PolygonFilter.instances`"""
        for p in PolygonFilter.instances:
            if p.unique_id == unique_id:
                PolygonFilter.instances.remove(p)


Instruction:
Given the following Python 3 function, write the documentation
def save(self, polyfile, ret_fobj=False):
        if is_file_obj(polyfile):
            fobj = polyfile
        else:
            fobj = pathlib.Path(polyfile).open("a")

        # Who the hell would use more then 10 million polygons or
        # polygon points? -> 08d (easier if other people want to import)
        data2write = []
        data2write.append("[Polygon {:08d}]".format(self.unique_id))
        data2write.append("X Axis = {}".format(self.axes[0]))
        data2write.append("Y Axis = {}".format(self.axes[1]))
        data2write.append("Name = {}".format(self.name))
        data2write.append("Inverted = {}".format(self.inverted))
        for i, point in enumerate(self.points):
            data2write.append("point{:08d} = {:.15e} {:.15e}".format(i,
                                                                     point[0],
                                                                     point[1]))
        # Add new lines
        for i in range(len(data2write)):
            data2write[i] += "\n"

        # begin writing to fobj
        fobj.writelines(data2write)

        if ret_fobj:
            return fobj
        else:
            fobj.close()

Response: Save all data to a text file.


Instruction:
How would you explain what the following Python 3 function does
def save_all(polyfile):
        nump = len(PolygonFilter.instances)
        if nump == 0:
            raise PolygonFilterError("There are not polygon filters to save.")
        for p in PolygonFilter.instances:
            # we return the ret_obj, so we don't need to open and
            # close the file multiple times.
            polyobj = p.save(polyfile, ret_fobj=True)
        polyobj.close()

Response: Save all polygon filters to a file.


Instruction:
Make a summary of the following Python 3 code
def get_resource(request):
    hash = request.matchdict['hash']

    # Do the file lookup
    with db_connect() as db_connection:
        with db_connection.cursor() as cursor:
            args = dict(hash=hash)
            cursor.execute(SQL['get-resource'], args)
            try:
                mimetype, file = cursor.fetchone()
            except TypeError:  # None returned
                raise httpexceptions.HTTPNotFound()

    resp = request.response
    resp.status = "200 OK"
    resp.content_type = mimetype
    resp.body = file[:]
    return resp

Response: Retrieve a file s data.


Instruction:
How would you implement a function in Python 3 that
returns an int value that contains the specified style bits set.

Response:
def get_style_bits(match=False, comment=False, selected=False, data=False, diff=False, user=0):
    """ Return an int value that contains the specified style bits set.

    Available styles for each byte are:

    match: part of the currently matched search
    comment: user commented area
    selected: selected region
    data: labeled in the disassembler as a data region (i.e. not disassembled)
    """
    style_bits = 0
    if user:
        style_bits |= (user & user_bit_mask)
    if diff:
        style_bits |= diff_bit_mask
    if match:
        style_bits |= match_bit_mask
    if comment:
        style_bits |= comment_bit_mask
    if data:
        style_bits |= (data_style & user_bit_mask)
    if selected:
        style_bits |= selected_bit_mask
    return style_bits


Instruction:
Can you generate the documentation for the following Python 3 function
def get_style_mask(**kwargs):
    bits = get_style_bits(**kwargs)
    if 'user' in kwargs and kwargs['user']:
        bits |= user_bit_mask
    else:
        bits &= (0xff ^ user_bit_mask)
    return 0xff ^ bits

Response:
Get the bit mask that will turn off the
    selected bits


Instruction:
Explain what the following Python 3 code does
def byte_bounds_offset(self):
        if self.data.base is None:
            if self.is_indexed:
                basearray = self.data.np_data
            else:
                basearray = self.data
            return 0, len(basearray)
        return int(self.data_start - self.base_start), int(self.data_end - self.base_start)

Response:
Return start and end offsets of this segment s data into the
            base array s data.


Instruction:
Create a Python 3 function for
getting index into base array s raw data given the index into this object s base array s base array

Response:
def get_raw_index(self, i):
        """Get index into base array's raw data, given the index into this
        segment
        """
        if self.is_indexed:
            return int(self.order[i])
        if self.data.base is None:
            return int(i)
        return int(self.data_start - self.base_start + i)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def get_indexes_from_base(self):
        if self.is_indexed:
            return np.copy(self.order[i])
        if self.data.base is None:
            i = 0
        else:
            i = self.get_raw_index(0)
        return np.arange(i, i + len(self), dtype=np.uint32)

Response:
Get array of indexes from the base array as if this raw data were were
            indexed.


Instruction:
Here you have a function in Python 3, explain what it does
def reverse_index_mapping(self):
        if self._reverse_index_mapping is None:
            if self.is_indexed:
                # Initialize array to out of range
                r = np.zeros(self.base_length, dtype=np.int32) - 1
                r[self.order] = np.arange(len(self.order), dtype=np.int32)
            elif self.data.base is None:
                # Starts at the beginning; produces the identity
                r = np.arange(self.data_length, dtype=np.int32)
            else:
                r = np.zeros(self.base_length, dtype=np.int32) - 1
                r[self.data_start - self.base_start:self.data_end - self.base_start] = np.arange(self.data_length, dtype=np.int32)
            self._reverse_index_mapping = r
        return self._reverse_index_mapping

Response: Get mapping from this segment s indexes to the indexes of the base array.


Instruction:
How would you implement a function in Python 3 that
gets the index into this segment s data given the index into the base data

Response:
def get_reverse_index(self, base_index):
        """Get index into this segment's data given the index into the base data

        Raises IndexError if the base index doesn't map to anything in this
        segment's data
        """
        r = self.reverse_index_mapping[base_index]
        if r < 0:
            raise IndexError("index %d not mapped in this segment" % base_index)
        return r


Instruction:
Can you tell what is the following Python 3 function doing
def resize(self, newsize, zeros=True):
        if not self.can_resize:
            raise ValueError("Segment %s can't be resized" % str(self))
        # only makes sense for the container (outermost) object
        if not self.rawdata.is_base:
            raise ValueError("Only container segments can be resized")
        origsize = len(self)
        self.rawdata.resize(newsize)
        self.set_raw(self.rawdata)  # force attributes to be reset
        newsize = len(self)
        if zeros:
            if newsize > origsize:
                self.data[origsize:] = 0
                self.style[origsize:] = 0
        return origsize, newsize

Response: Resize the data arrays of the child segments to the new size.


Instruction:
Can you tell what is the following Python 3 function doing
def restore_renamed_serializable_attributes(self):
        if hasattr(self, 'start_addr'):
            self.origin = self.start_addr
            log.debug(f"moving start_addr to origin: {self.start_addr}")
            delattr(self, 'start_addr')

Response: Hook for the future if attributes have been renamed.


Instruction:
Can you tell what is the following Python 3 function doing
def reconstruct_raw(self, rawdata):
        start, end = self._rawdata_bounds
        r = rawdata[start:end]
        delattr(self, '_rawdata_bounds')
        try:
            if self._order_list:
                order = to_numpy_list(self._order_list)
                r = r.get_indexed(order)
                delattr(self, '_order_list')
        except AttributeError:
            pass
        self.set_raw(r)

Response: Reconstruct the pointers to the parent data arrays


Instruction:
Can you implement a function in Python 3 that
gets the raw data that is similar to the specified other segment.

Response:
def get_parallel_raw_data(self, other):
        """ Get the raw data that is similar to the specified other segment
        """
        start, end = other.byte_bounds_offset()
        r = self.rawdata[start:end]
        if other.rawdata.is_indexed:
            r = r.get_indexed[other.order]
        return r


Instruction:
Given the following Python 3 function, write the documentation
def serialize_session(self, mdict):
        mdict["comment ranges"] = [list(a) for a in self.get_style_ranges(comment=True)]
        mdict["data ranges"] = [list(a) for a in self.get_style_ranges(data=True)]
        for i in range(1, user_bit_mask):
            r = [list(a) for a in self.get_style_ranges(user=i)]
            if r:
                slot = "user style %d" % i
                mdict[slot] = r

        # json serialization doesn't allow int keys, so convert to list of
        # pairs
        mdict["comments"] = self.get_sorted_comments()

Response: Save extra metadata to a dict so that it can be serialized.


Instruction:
Write a Python 3 function for
getting index into this array s data given the index into the base array

Response:
def get_index_from_base_index(self, base_index):
        """Get index into this array's data given the index into the base array
        """
        r = self.rawdata
        try:
            index = r.get_reverse_index(base_index)
        except IndexError:
            raise IndexError("index %d not in this segment" % base_index)
        if index < 0:
            raise IndexError("index %d not in this segment" % base_index)
        return int(index)


Instruction:
Write a Python 3 function for
returning a list of start end pairs that match the specified style

Response:
def get_style_ranges(self, **kwargs):
        """Return a list of start, end pairs that match the specified style
        """
        style_bits = self.get_style_bits(**kwargs)
        matches = (self.style & style_bits) == style_bits
        return self.bool_to_ranges(matches)


Instruction:
Here you have a function in Python 3, explain what it does
def fixup_comments(self):
        style_base = self.rawdata.style_base
        comment_text_indexes = np.asarray(list(self.rawdata.extra.comments.keys()), dtype=np.uint32)
        comment_mask = self.get_style_mask(comment=True)
        has_comments = np.where(style_base & comment_bit_mask > 0)[0]
        both = np.intersect1d(comment_text_indexes, has_comments)
        log.info("fixup comments: %d correctly marked, %d without style, %d empty text" % (np.alen(both), np.alen(comment_text_indexes) - np.alen(both), np.alen(has_comments) - np.alen(both)))
        style_base &= comment_mask
        comment_style = self.get_style_bits(comment=True)
        style_base[comment_text_indexes] |= comment_style

Response: Fixup any style bytes that are marked as commented but have no comments.


Instruction:
Write a Python 3 function for
finding sections of the segment that have the same style value. The arguments to this function are used as a mask for the style to determine where to split the styles. Style bits that aren't included in the list will be ignored when splitting. The returned list covers the entire length of the segment. Returns a list of tuples, each tuple containing two items: a start, end tuple; and an integer with the style value.

Response:
def get_entire_style_ranges(self, split_comments=None, **kwargs):
        """Find sections of the segment that have the same style value.

        The arguments to this function are used as a mask for the style to
        determine where to split the styles. Style bits that aren't included in
        the list will be ignored when splitting. The returned list covers the
        entire length of the segment.

        Returns a list of tuples, each tuple containing two items: a start, end
        tuple; and an integer with the style value.
        """
        style_bits = self.get_style_bits(**kwargs)
        matches = self.get_comment_locations(**kwargs)
        groups = np.split(matches, np.where(np.diff(matches) != 0)[0] + 1)
        if split_comments is None:
            split_comments = []
        # print groups
        # split into groups with the same numbers
        ranges = []
        last_end = 0
        if len(groups) == 1 and len(groups[0]) == 0:
            # check for degenerate case
            return
        last_style = -1
        for group in groups:
            # each group is guaranteed to have the same style
            size = len(group)
            next_end = last_end + size
            style = matches[last_end]
            masked_style = style & style_bits
            # print last_end, next_end, style, masked_style, size, group
            if style & comment_bit_mask:
                if masked_style in split_comments:
                    # print "interesting comment", last_end, next_end
                    ranges.append(((last_end, next_end), masked_style))
                else:
                    # print "non-interesting comment", last_end, next_end
                    if last_style == masked_style:
                        ((prev_end, _), _) = ranges.pop()
                        ranges.append(((prev_end, next_end), masked_style))
                    else:
                        ranges.append(((last_end, next_end), masked_style))
            else:
                if last_style == masked_style:
                    ((prev_end, _), _) = ranges.pop()
                    ranges.append(((prev_end, next_end), masked_style))
                else:
                    ranges.append(((last_end, next_end), masked_style))
            last_style = masked_style
            last_end = next_end
        return ranges


Instruction:
How would you explain what the following Python 3 function does
def get_comments_at_indexes(self, indexes):
        s = self.style[indexes]
        has_comments = np.where(s & comment_bit_mask > 0)[0]
        comments = []
        for where_index in has_comments:
            raw = self.get_raw_index(indexes[where_index])
            try:
                comment = self.rawdata.extra.comments[raw]
            except KeyError:
                comment = None
            comments.append(comment)
        return has_comments, comments

Response: Get a list of comments at specified indexes


Instruction:
Explain what the following Python 3 code does
def get_comment_restore_data(self, ranges):
        restore_data = []
        for start, end in ranges:
            log.debug("range: %d-%d" % (start, end))
            styles = self.style[start:end].copy()
            items = {}
            for i in range(start, end):
                rawindex = self.get_raw_index(i)
                try:
                    comment = self.rawdata.extra.comments[rawindex]
                    log.debug("  index: %d rawindex=%d '%s'" % (i, rawindex, comment))
                    items[i] = (rawindex, comment)
                except KeyError:
                    log.debug("  index: %d rawindex=%d NO COMMENT TO SAVE" % (i, rawindex))
                    items[i] = (rawindex, None)

            restore_data.append((start, end, styles, items))
        return restore_data

Response: Get a chunk of data that can be used to recreate the comments on an undo ArcGIS undo ArcGIS undo ArcGIS undo ArcGIS undo ArcGIS undo ArcGIS undo ArcGIS undo ArcGIS undo ArcGIS undo ArcGIS undo ArcGIS undo ArcGIS undo ArcGIS undo ArcGIS undo ArcGIS undo ArcGIS undo ArcGIS undo ArcGIS undo ArcGIS undo ArcGIS undo ArcGIS