Instruction:
How would you explain what the following Python 3 function does
def _add_countriesdata(cls, iso3, country):
        # type: (str, hxl.Row) -> None
        """
        Set up countries data from data in form provided by UNStats and World Bank

        Args:
            iso3 (str): ISO3 code for country
            country (hxl.Row): Country information

        Returns:
            None
        """
        countryname = country.get('#country+name+preferred')
        cls._countriesdata['countrynames2iso3'][countryname.upper()] = iso3
        iso2 = country.get('#country+code+v_iso2')
        if iso2:
            cls._countriesdata['iso2iso3'][iso2] = iso3
            # different types so keys won't clash
            cls._countriesdata['iso2iso3'][iso3] = iso2
        m49 = country.get('#country+code+num+v_m49')
        if m49:
            m49 = int(m49)
            cls._countriesdata['m49iso3'][m49] = iso3
            # different types so keys won't clash
            cls._countriesdata['m49iso3'][iso3] = m49
        cls._countriesdata['aliases'][iso3] = re.compile(country.get('#country+regex'), re.IGNORECASE)
        regionname = country.get('#region+main+name+preferred')
        sub_regionname = country.get('#region+sub+name+preferred')
        intermediate_regionname = country.get('#region+intermediate+name+preferred')
        regionid = country.get('#region+main+code')
        if regionid:
            regionid = int(regionid)
        sub_regionid = country.get('#region+sub+code')
        if sub_regionid:
            sub_regionid = int(sub_regionid)
        intermediate_regionid = country.get('#region+intermediate+code')
        if intermediate_regionid:
            intermediate_regionid = int(intermediate_regionid)

        # region, subregion and intermediate region codes do not clash so only need one dict
        def add_country_to_set(colname, idval, iso3):
            value = cls._countriesdata[colname].get(idval)
            if value is None:
                value = set()
                cls._countriesdata['regioncodes2countries'][idval] = value
            value.add(iso3)

        if regionname:
            add_country_to_set('regioncodes2countries', regionid, iso3)
            cls._countriesdata['regioncodes2names'][regionid] = regionname
            cls._countriesdata['regionnames2codes'][regionname.upper()] = regionid
        if sub_regionname:
            add_country_to_set('regioncodes2countries', sub_regionid, iso3)
            cls._countriesdata['regioncodes2names'][sub_regionid] = sub_regionname
            cls._countriesdata['regionnames2codes'][sub_regionname.upper()] = sub_regionid
        if intermediate_regionname:
            add_country_to_set('regioncodes2countries', intermediate_regionid, iso3)
            cls._countriesdata['regioncodes2names'][intermediate_regionid] = intermediate_regionname
            cls._countriesdata['regionnames2codes'][intermediate_regionname.upper()] = \
                intermediate_regionid

Response: Add countries data from country information to the internal dictionary.


Instruction:
Here you have a function in Python 3, explain what it does
def set_countriesdata(cls, countries):
        # type: (str) -> None
        """
        Set up countries data from data in form provided by UNStats and World Bank

        Args:
            countries (str): Countries data in HTML format provided by UNStats

        Returns:
            None
        """
        cls._countriesdata = dict()
        cls._countriesdata['countries'] = dict()
        cls._countriesdata['iso2iso3'] = dict()
        cls._countriesdata['m49iso3'] = dict()
        cls._countriesdata['countrynames2iso3'] = dict()
        cls._countriesdata['regioncodes2countries'] = dict()
        cls._countriesdata['regioncodes2names'] = dict()
        cls._countriesdata['regionnames2codes'] = dict()
        cls._countriesdata['aliases'] = dict()

        for country in countries:
            iso3 = country.get('#country+code+v_iso3')
            if not iso3:
                continue
            iso3 = iso3.upper()
            cls._add_countriesdata(iso3, country)
            cls._countriesdata['countries'][iso3] = country.dictionary

        def sort_list(colname):
            for idval in cls._countriesdata[colname]:
                cls._countriesdata[colname][idval] = \
                    sorted(list(cls._countriesdata[colname][idval]))

        sort_list('regioncodes2countries')

Response: Sets up countries data from data provided by UNStats and World Bank.


Instruction:
Can you implement a function in Python 3 that
reads countries data from OCHA and stores it in the class s _countriesdata attribute.

Response:
def countriesdata(cls, use_live=True):
        # type: (bool) -> List[Dict[Dict]]
        """
        Read countries data from OCHA countries feed (falling back to file)

        Args:
            use_live (bool): Try to get use latest data from web rather than file in package. Defaults to True.

        Returns:
            List[Dict[Dict]]: Countries dictionaries
        """
        if cls._countriesdata is None:
            countries = None
            if use_live:
                try:
                    countries = hxl.data(cls._ochaurl)
                except IOError:
                    logger.exception('Download from OCHA feed failed! Falling back to stored file.')
            if countries is None:
                countries = hxl.data(
                    script_dir_plus_file('Countries & Territories Taxonomy MVP - C&T Taxonomy with HXL Tags.csv',
                                         Country), allow_local=True)
            cls.set_countriesdata(countries)
        return cls._countriesdata


Instruction:
How would you implement a function in Python 3 that
sets the World Bank url from which to retrieve countries data.

Response:
def set_ocha_url(cls, url=None):
        # type: (str) -> None
        """
        Set World Bank url from which to retrieve countries data

        Args:
            url (str): World Bank url from which to retrieve countries data. Defaults to internal value.

        Returns:
            None
        """
        if url is None:
            url = cls._ochaurl_int
        cls._ochaurl = url


Instruction:
How would you code a function in Python 3 to
return the country information for a given ISO3 code.

Response:
def get_country_info_from_iso3(cls, iso3, use_live=True, exception=None):
        # type: (str, bool, Optional[ExceptionUpperBound]) -> Optional[Dict[str]]
        """Get country information from ISO3 code

        Args:
            iso3 (str): ISO3 code for which to get country information
            use_live (bool): Try to get use latest data from web rather than file in package. Defaults to True.
            exception (Optional[ExceptionUpperBound]): An exception to raise if country not found. Defaults to None.

        Returns:
            Optional[Dict[str]]: country information
        """
        countriesdata = cls.countriesdata(use_live=use_live)
        country = countriesdata['countries'].get(iso3.upper())
        if country is not None:
            return country

        if exception is not None:
            raise exception
        return None


Instruction:
Implement a Python 3 function for
returning the name of the country from ISO3 code.

Response:
def get_country_name_from_iso3(cls, iso3, use_live=True, exception=None):
        # type: (str, bool, Optional[ExceptionUpperBound]) -> Optional[str]
        """Get country name from ISO3 code

        Args:
            iso3 (str): ISO3 code for which to get country name
            use_live (bool): Try to get use latest data from web rather than file in package. Defaults to True.
            exception (Optional[ExceptionUpperBound]): An exception to raise if country not found. Defaults to None.

        Returns:
            Optional[str]: Country name
        """
        countryinfo = cls.get_country_info_from_iso3(iso3, use_live=use_live, exception=exception)
        if countryinfo is not None:
            return countryinfo.get('#country+name+preferred')
        return None


Instruction:
Can you generate a brief explanation for the following Python 3 code
def get_iso2_from_iso3(cls, iso3, use_live=True, exception=None):
        # type: (str, bool, Optional[ExceptionUpperBound]) -> Optional[str]
        """Get ISO2 from ISO3 code

        Args:
            iso3 (str): ISO3 code for which to get ISO2 code
            use_live (bool): Try to get use latest data from web rather than file in package. Defaults to True.
            exception (Optional[ExceptionUpperBound]): An exception to raise if country not found. Defaults to None.

        Returns:
            Optional[str]: ISO2 code
        """
        countriesdata = cls.countriesdata(use_live=use_live)
        iso2 = countriesdata['iso2iso3'].get(iso3.upper())
        if iso2 is not None:
            return iso2

        if exception is not None:
            raise exception
        return None

Response: Get ISO2 code from ISO3 code.


Instruction:
Can you tell what is the following Python 3 function doing
def get_country_info_from_iso2(cls, iso2, use_live=True, exception=None):
        # type: (str, bool, Optional[ExceptionUpperBound]) -> Optional[Dict[str]]
        """Get country name from ISO2 code

        Args:
            iso2 (str): ISO2 code for which to get country information
            use_live (bool): Try to get use latest data from web rather than file in package. Defaults to True.
            exception (Optional[ExceptionUpperBound]): An exception to raise if country not found. Defaults to None.

        Returns:
            Optional[Dict[str]]: Country information
        """
        iso3 = cls.get_iso3_from_iso2(iso2, use_live=use_live, exception=exception)
        if iso3 is not None:
            return cls.get_country_info_from_iso3(iso3, use_live=use_live, exception=exception)
        return None

Response: Returns the country name from ISO2 code.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def get_country_name_from_iso2(cls, iso2, use_live=True, exception=None):
        # type: (str, bool, Optional[ExceptionUpperBound]) -> Optional[str]
        """Get country name from ISO2 code

        Args:
            iso2 (str): ISO2 code for which to get country name
            use_live (bool): Try to get use latest data from web rather than file in package. Defaults to True.
            exception (Optional[ExceptionUpperBound]): An exception to raise if country not found. Defaults to None.

        Returns:
            Optional[str]: Country name
        """
        iso3 = cls.get_iso3_from_iso2(iso2, use_live=use_live, exception=exception)
        if iso3 is not None:
            return cls.get_country_name_from_iso3(iso3, exception=exception)
        return None

Response: Returns the name of the country from ISO2 code.


Instruction:
How would you implement a function in Python 3 that
gets M49 code from ISO3 code.

Response:
def get_m49_from_iso3(cls, iso3, use_live=True, exception=None):
        # type: (str, bool, Optional[ExceptionUpperBound]) -> Optional[int]
        """Get M49 from ISO3 code

        Args:
            iso3 (str): ISO3 code for which to get M49 code
            use_live (bool): Try to get use latest data from web rather than file in package. Defaults to True.
            exception (Optional[ExceptionUpperBound]): An exception to raise if country not found. Defaults to None.

        Returns:
            Optional[int]: M49 code
        """
        countriesdata = cls.countriesdata(use_live=use_live)
        m49 = countriesdata['m49iso3'].get(iso3)
        if m49 is not None:
            return m49

        if exception is not None:
            raise exception
        return None


Instruction:
Can you generate the documentation for the following Python 3 function
def get_country_info_from_m49(cls, m49, use_live=True, exception=None):
        # type: (int, bool, Optional[ExceptionUpperBound]) -> Optional[Dict[str]]
        """Get country name from M49 code

        Args:
            m49 (int): M49 numeric code for which to get country information
            use_live (bool): Try to get use latest data from web rather than file in package. Defaults to True.
            exception (Optional[ExceptionUpperBound]): An exception to raise if country not found. Defaults to None.

        Returns:
            Optional[Dict[str]]: Country information
        """
        iso3 = cls.get_iso3_from_m49(m49, use_live=use_live, exception=exception)
        if iso3 is not None:
            return cls.get_country_info_from_iso3(iso3, exception=exception)
        return None

Response: Get country name from M49 code.


Instruction:
Here you have a function in Python 3, explain what it does
def get_country_name_from_m49(cls, m49, use_live=True, exception=None):
        # type: (int, bool, Optional[ExceptionUpperBound]) -> Optional[str]
        """Get country name from M49 code

        Args:
            m49 (int): M49 numeric code for which to get country name
            use_live (bool): Try to get use latest data from web rather than file in package. Defaults to True.
            exception (Optional[ExceptionUpperBound]): An exception to raise if country not found. Defaults to None.

        Returns:
            Optional[str]: Country name
        """
        iso3 = cls.get_iso3_from_m49(m49, use_live=use_live, exception=exception)
        if iso3 is not None:
            return cls.get_country_name_from_iso3(iso3, exception=exception)
        return None

Response: Returns the country name from M49 code.


Instruction:
Make a summary of the following Python 3 code
def expand_countryname_abbrevs(cls, country):
        # type: (str) -> List[str]
        """Expands abbreviation(s) in country name in various ways (eg. FED -> FEDERATED, FEDERAL etc.)

        Args:
            country (str): Country with abbreviation(s)to expand

        Returns:
            List[str]: Uppercase country name with abbreviation(s) expanded in various ways
        """
        def replace_ensure_space(word, replace, replacement):
            return word.replace(replace, '%s ' % replacement).replace('  ', ' ').strip()
        countryupper = country.upper()
        for abbreviation in cls.abbreviations:
            countryupper = replace_ensure_space(countryupper, abbreviation, cls.abbreviations[abbreviation])
        candidates = [countryupper]
        for abbreviation in cls.multiple_abbreviations:
            if abbreviation in countryupper:
                for expanded in cls.multiple_abbreviations[abbreviation]:
                    candidates.append(replace_ensure_space(countryupper, abbreviation, expanded))
        return candidates

Response: Expands abbreviation in country name in various ways.


Instruction:
How would you explain what the following Python 3 function does
def simplify_countryname(cls, country):
        # type: (str) -> (str, List[str])
        """Simplifies country name by removing descriptive text eg. DEMOCRATIC, REPUBLIC OF etc.

        Args:
            country (str): Country name to simplify

        Returns:
            Tuple[str, List[str]]: Uppercase simplified country name and list of removed words
        """
        countryupper = country.upper()
        words = get_words_in_sentence(countryupper)
        index = countryupper.find(',')
        if index != -1:
            countryupper = countryupper[:index]
        index = countryupper.find(':')
        if index != -1:
            countryupper = countryupper[:index]
        regex = re.compile('\(.+?\)')
        countryupper = regex.sub('', countryupper)
        remove = copy.deepcopy(cls.simplifications)
        for simplification1, simplification2 in cls.abbreviations.items():
            countryupper = countryupper.replace(simplification1, '')
            remove.append(simplification2)
        for simplification1, simplifications in cls.multiple_abbreviations.items():
            countryupper = countryupper.replace(simplification1, '')
            for simplification2 in simplifications:
                remove.append(simplification2)
        remove = '|'.join(remove)
        regex = re.compile(r'\b(' + remove + r')\b', flags=re.IGNORECASE)
        countryupper = regex.sub('', countryupper)
        countryupper = countryupper.strip()
        countryupper_words = get_words_in_sentence(countryupper)
        if len(countryupper_words) > 1:
            countryupper = countryupper_words[0]
        if countryupper:
            words.remove(countryupper)
        return countryupper, words

Response: Simplifies a country name by removing descriptive text eg. DEMOCRATIC REPUBLIC OF etc.


Instruction:
Implement a function in Python 3 to
return ISO3 code for a given country. Only exact matches or None are returned.

Response:
def get_iso3_country_code(cls, country, use_live=True, exception=None):
        # type: (str, bool, Optional[ExceptionUpperBound]) -> Optional[str]
        """Get ISO3 code for cls. Only exact matches or None are returned.

        Args:
            country (str): Country for which to get ISO3 code
            use_live (bool): Try to get use latest data from web rather than file in package. Defaults to True.
            exception (Optional[ExceptionUpperBound]): An exception to raise if country not found. Defaults to None.

        Returns:
            Optional[str]: ISO3 country code or None
        """
        countriesdata = cls.countriesdata(use_live=use_live)
        countryupper = country.upper()
        len_countryupper = len(countryupper)
        if len_countryupper == 3:
            if countryupper in countriesdata['countries']:
                return countryupper
        elif len_countryupper == 2:
            iso3 = countriesdata['iso2iso3'].get(countryupper)
            if iso3 is not None:
                return iso3

        iso3 = countriesdata['countrynames2iso3'].get(countryupper)
        if iso3 is not None:
            return iso3

        for candidate in cls.expand_countryname_abbrevs(countryupper):
            iso3 = countriesdata['countrynames2iso3'].get(candidate)
            if iso3 is not None:
                return iso3

        if exception is not None:
            raise exception
        return None


Instruction:
Can you generate a brief explanation for the following Python 3 code
def get_iso3_country_code_fuzzy(cls, country, use_live=True, exception=None):
        # type: (str, bool, Optional[ExceptionUpperBound]) -> Tuple[[Optional[str], bool]]
        """Get ISO3 code for cls. A tuple is returned with the first value being the ISO3 code and the second
        showing if the match is exact or not.

        Args:
            country (str): Country for which to get ISO3 code
            use_live (bool): Try to get use latest data from web rather than file in package. Defaults to True.
            exception (Optional[ExceptionUpperBound]): An exception to raise if country not found. Defaults to None.

        Returns:
            Tuple[[Optional[str], bool]]: ISO3 code and if the match is exact or (None, False).
        """
        countriesdata = cls.countriesdata(use_live=use_live)
        iso3 = cls.get_iso3_country_code(country,
                                         use_live=use_live)  # don't put exception param here as we don't want it to throw

        if iso3 is not None:
            return iso3, True

        def remove_matching_from_list(wordlist, word_or_part):
            for word in wordlist:
                if word_or_part in word:
                    wordlist.remove(word)

        # fuzzy matching
        expanded_country_candidates = cls.expand_countryname_abbrevs(country)
        match_strength = 0
        matches = set()
        for countryname in sorted(countriesdata['countrynames2iso3']):
            for candidate in expanded_country_candidates:
                simplified_country, removed_words = cls.simplify_countryname(candidate)
                if simplified_country in countryname:
                    words = get_words_in_sentence(countryname)
                    new_match_strength = 0
                    if simplified_country:
                        remove_matching_from_list(words, simplified_country)
                        new_match_strength += 32
                    for word in removed_words:
                        if word in countryname:
                            remove_matching_from_list(words, word)
                            new_match_strength += 4
                        else:
                            if word in cls.major_differentiators:
                                new_match_strength -= 16
                            else:
                                new_match_strength -= 1
                    for word in words:
                        if word in cls.major_differentiators:
                            new_match_strength -= 16
                        else:
                            new_match_strength -= 1
                    iso3 = countriesdata['countrynames2iso3'][countryname]
                    if new_match_strength > match_strength:
                        match_strength = new_match_strength
                        matches = set()
                    if new_match_strength == match_strength:
                        matches.add(iso3)

        if len(matches) == 1 and match_strength > 16:
            return matches.pop(), False

        # regex lookup
        for iso3, regex in countriesdata['aliases'].items():
            index = re.search(regex, country.upper())
            if index is not None:
                return iso3, False

        if exception is not None:
            raise exception
        return None, False

Response: Returns the ISO3 code for a given country.


Instruction:
Can you implement a function in Python 3 that
returns a list of countries in a given region

Response:
def get_countries_in_region(cls, region, use_live=True, exception=None):
        # type: (Union[int,str], bool, Optional[ExceptionUpperBound]) -> List[str]
        """Get countries (ISO3 codes) in region

        Args:
            region (Union[int,str]): Three digit UNStats M49 region code or region name
            use_live (bool): Try to get use latest data from web rather than file in package. Defaults to True.
            exception (Optional[ExceptionUpperBound]): An exception to raise if region not found. Defaults to None.

        Returns:
            List(str): Sorted list of ISO3 country names
        """
        countriesdata = cls.countriesdata(use_live=use_live)
        if isinstance(region, int):
            regioncode = region
        else:
            regionupper = region.upper()
            regioncode = countriesdata['regionnames2codes'].get(regionupper)

        if regioncode is not None:
            return countriesdata['regioncodes2countries'][regioncode]

        if exception is not None:
            raise exception
        return list()


Instruction:
Implement a Python 3 function for
loading the profile of samples.

Response:
def load_profile(ctx, variant_file, update, stats, profile_threshold):

    """
        Command for profiling of samples. User may upload variants used in profiling
        from a vcf, update the profiles for all samples, and get some stats
        from the profiles in the database.

        Profiling is used to monitor duplicates in the database. The profile is
        based on the variants in the 'profile_variant' collection, assessing
        the genotypes for each sample at the position of these variants.
    """

    adapter = ctx.obj['adapter']

    if variant_file:
        load_profile_variants(adapter, variant_file)

    if update:
        update_profiles(adapter)

    if stats:

        distance_dict = profile_stats(adapter, threshold=profile_threshold)
        click.echo(table_from_dict(distance_dict))


Instruction:
Can you write a function in Python 3 where it
adds several variants to the profile_variant collection in the database

Response:
def add_profile_variants(self, profile_variants):

        """Add several variants to the profile_variant collection in the
        database

        Args:

            profile_variants(list(models.ProfileVariant))

        """

        results = self.db.profile_variant.insert_many(profile_variants)

        return results


Instruction:
Can you write a function in Python 3 where it
returns the ZIP parameters that best fit a given data set.

Response:
def zip_fit_params(data):
    """
    Returns the ZIP parameters that best fit a given data set.

    Args:
        data (array): 2d array of genes x cells belonging to a given cluster

    Returns:
        L (array): 1d array of means
        M (array): 1d array of zero-inflation parameter
    """
    genes, cells = data.shape
    m = data.mean(1)
    v = data.var(1)
    M = (v-m)/(m**2+v-m)
    #M = v/(v+m**2)
    #M[np.isnan(M)] = 0.0
    M = np.array([min(1.0, max(0.0, x)) for x in M])
    L = m + v/m - 1.0
    #L = (v + m**2)/m
    L[np.isnan(L)] = 0.0
    L = np.array([max(0.0, x) for x in L])
    return L, M


Instruction:
Write a Python 3 script to
perform hard EM clustering using the zero - inflated Poisson distribution.

Response:
def zip_cluster(data, k, init=None, max_iters=100):
    """
    Performs hard EM clustering using the zero-inflated Poisson distribution.

    Args:
        data (array): A 2d array- genes x cells
        k (int): Number of clusters
        init (array, optional): Initial centers - genes x k array. Default: None, use kmeans++
        max_iters (int, optional): Maximum number of iterations. Default: 100

    Returns:
        assignments (array): integer assignments of cells to clusters (length cells)
        L (array): Poisson parameter (genes x k)
        M (array): zero-inflation parameter (genes x k)
    """
    genes, cells = data.shape
    init, new_assignments = kmeans_pp(data+eps, k, centers=init)
    centers = np.copy(init)
    M = np.zeros(centers.shape)
    assignments = new_assignments
    for c in range(k):
        centers[:,c], M[:,c] = zip_fit_params_mle(data[:, assignments==c])
    for it in range(max_iters):
        lls = zip_ll(data, centers, M)
        new_assignments = np.argmax(lls, 1)
        if np.equal(assignments, new_assignments).all():
            return assignments, centers, M
        for c in range(k):
            centers[:,c], M[:,c] = zip_fit_params_mle(data[:, assignments==c])
        assignments = new_assignments
    return assignments, centers, M


Instruction:
Can you tell what is the following Python 3 function doing
def diffusion_mds(means, weights, d, diffusion_rounds=10):
    for i in range(diffusion_rounds):
        weights = weights*weights
        weights = weights/weights.sum(0)
    X = dim_reduce(means, weights, d)
    if X.shape[0]==2:
        return X.dot(weights)
    else:
        return X.T.dot(weights)

Response: Dimensionality reduction using MDS


Instruction:
Make a summary of the following Python 3 code
def mds(means, weights, d):
    X = dim_reduce(means, weights, d)
    if X.shape[0]==2:
        return X.dot(weights)
    else:
        return X.T.dot(weights)

Response: Dimensionality reduction using MDS.


Instruction:
Can you generate the documentation for the following Python 3 function
def dim_reduce_data(data, d):
    genes, cells = data.shape
    distances = np.zeros((cells, cells))
    for i in range(cells):
        for j in range(cells):
            distances[i,j] = poisson_dist(data[:,i], data[:,j])
    # do MDS on the distance matrix (procedure from Wikipedia)
    proximity = distances**2
    J = np.eye(cells) - 1./cells
    B = -0.5*np.dot(J, np.dot(proximity, J))
    # B should be symmetric, so we can use eigh
    e_val, e_vec = np.linalg.eigh(B)
    # Note: lam should be ordered to be the largest eigenvalues
    lam = np.diag(e_val[-d:])[::-1]
    #lam = max_or_zero(lam)
    E = e_vec[:,-d:][::-1]
    X = np.dot(E, lam**0.5)
    return X

Response: Does a MDS on the data directly not on the means.


Instruction:
Write a Python 3 script for
getting a case from the database

Response:
def case(self, case):
        """Get a case from the database

            Search the cases with the case id

            Args:
                case (dict): A case dictionary

            Returns:
                mongo_case (dict): A mongo case dictionary
        """
        LOG.debug("Getting case {0} from database".format(case.get('case_id')))
        case_id = case['case_id']
        return self.db.case.find_one({'case_id': case_id})


Instruction:
Can you tell what is the following Python 3 function doing
def nr_cases(self, snv_cases=None, sv_cases=None):
        query = {}

        if snv_cases:
            query = {'vcf_path': {'$exists':True}}
        if sv_cases:
            query = {'vcf_sv_path': {'$exists':True}}
        if snv_cases and sv_cases:
            query = None

        return self.db.case.count_documents(query)

Response: Return the number of cases in the database


Instruction:
How would you implement a function in Python 3 that
adds a case to the case collection

Response:
def add_case(self, case, update=False):
        """Add a case to the case collection

        If the case exists and update is False raise error.

        Args:
            db (MongoClient): A connection to the mongodb
            case (dict): A case dictionary
            update(bool): If existing case should be updated

        Returns:
            mongo_case_id(ObjectId)

        """
        existing_case = self.case(case)
        if existing_case and not update:
            raise CaseError("Case {} already exists".format(case['case_id']))
        if existing_case:
            self.db.case.find_one_and_replace(
                {'case_id': case['case_id']},
                case,
            )
        else:
            self.db.case.insert_one(case)

        return case


Instruction:
Can you create a Python 3 function that
deletes a case from the database

Response:
def delete_case(self, case):
        """Delete case from the database

            Delete a case from the database

            Args:
                case (dict): A case dictionary

        """
        mongo_case = self.case(case)

        if not mongo_case:
            raise CaseError("Tried to delete case {0} but could not find case".format(
                case.get('case_id')
            ))
        LOG.info("Removing case {0} from database".format(
            mongo_case.get('case_id')
        ))
        self.db.case.delete_one({'_id': mongo_case['_id']})

        return


Instruction:
How would you explain what the following Python 3 function does
def make_sink_proxies(drivers):
    ''' Build a list of sink proxies. sink proxies are a two-level ordered
    dictionary. The first level contains the lst of drivers, and the second
    level contains the list of sink proxies for each driver:

    drv1-->sink1
      | |->sink2
      |
    drv2-->sink1
        |->sink2
    '''
    sink_proxies = OrderedDict()
    if drivers is not None:
        for driver_name in drivers._fields:
            driver = getattr(drivers, driver_name)
            driver_sink = getattr(driver, 'input')
            if driver_sink is not None:
                driver_sink_proxies = OrderedDict()
                for name in driver_sink._fields:
                    driver_sink_proxies[name] = Subject()

                sink_proxies[driver_name] = driver.input(**driver_sink_proxies)
    return sink_proxies

Response: Build a list of sink proxies for each driver.


Instruction:
Here you have a function in Python 3, explain what it does
def build_profile_variant(variant):

    chrom = variant.CHROM
    if chrom.startswith(('chr', 'CHR', 'Chr')):
        chrom = chrom[3:]

    pos = int(variant.POS)

    variant_id = get_variant_id(variant)

    ref = variant.REF
    alt = variant.ALT[0]

    maf = get_maf(variant)

    profile_variant = ProfileVariant(
        variant_id=variant_id,
        chrom=chrom,
        pos=pos,
        ref=ref,
        alt=alt,
        maf=maf,
        id_column = variant.ID
    )

    return profile_variant

Response: Builds a ProfileVariant object from a Cyvcf2. Variant object.


Instruction:
Given the following Python 3 function, write the documentation
def add_headers(vcf_obj, nr_cases=None, sv=False):

    vcf_obj.add_info_to_header(
        {
            'ID':"Obs",
            'Number': '1',
            'Type': 'Integer',
            'Description': "The number of observations for the variant"}
    )
    if not sv:
        vcf_obj.add_info_to_header(
            {
                'ID':"Hom",
                'Number': '1',
                'Type': 'Integer',
                'Description': "The number of observed homozygotes"}
        )
        vcf_obj.add_info_to_header(
            {
                'ID':"Hem",
                'Number': '1',
                'Type': 'Integer',
                'Description': "The number of observed hemizygotes"}
        )
    if nr_cases:
        case_header = "##NrCases={}".format(nr_cases)
        vcf_obj.add_to_header(case_header)
    # head.add_version_tracking("loqusdb", version, datetime.now().strftime("%Y-%m-%d %H:%M"))
    return

Response: Add loqus specific information to a VCF header


Instruction:
Write a Python 3 script for
returning a VCF object for a given file path.

Response:
def get_file_handle(file_path):
    """Return cyvcf2 VCF object

    Args:
        file_path(str)

    Returns:
        vcf_obj(cyvcf2.VCF)
    """
    LOG.debug("Check if file end is correct")

    if not os.path.exists(file_path):
        raise IOError("No such file:{0}".format(file_path))

    if not os.path.splitext(file_path)[-1] in VALID_ENDINGS:
        raise IOError("Not a valid vcf file name: {}".format(file_path))

    vcf_obj = VCF(file_path)

    return vcf_obj


Instruction:
Implement a Python 3 function for
checking if the vcf file is in correct format.

Response:
def check_vcf(vcf_path, expected_type='snv'):
    """Check if there are any problems with the vcf file

    Args:
        vcf_path(str)
        expected_type(str): 'sv' or 'snv'

    Returns:
        vcf_info(dict): dict like
        {
            'nr_variants':<INT>,
            'variant_type': <STR> in ['snv', 'sv'],
            'individuals': <LIST> individual positions in file
        }
    """
    LOG.info("Check if vcf is on correct format...")

    vcf = VCF(vcf_path)
    individuals = vcf.samples
    variant_type = None

    previous_pos = None
    previous_chrom = None

    posititon_variants = set()

    nr_variants = 0
    for nr_variants,variant in enumerate(vcf,1):

        # Check the type of variant
        current_type = 'sv' if variant.var_type == 'sv' else 'snv'
        if not variant_type:
            variant_type = current_type

        # Vcf can not include both snvs and svs
        if variant_type != current_type:
            raise VcfError("Vcf includes a mix of snvs and svs")

        current_chrom = variant.CHROM
        current_pos = variant.POS

        # We start with a simple id that can be used by SV:s
        variant_id = "{0}_{1}".format(current_chrom, current_pos)
        # For SNVs we can create a proper variant id with chrom_pos_ref_alt
        if variant_type == 'snv':
            variant_id = get_variant_id(variant)

        # Initiate variables
        if not previous_chrom:
            previous_chrom = current_chrom
            previous_pos = current_pos
            posititon_variants = set([variant_id])
            continue

        # Update variables if new chromosome
        if current_chrom != previous_chrom:
            previous_chrom = current_chrom
            previous_pos = current_pos
            posititon_variants = set([variant_id])
            continue

        if variant_type == 'snv':
            # Check if variant is unique
            if current_pos == previous_pos:
                if variant_id in posititon_variants:
                    raise VcfError("Variant {0} occurs several times"\
                                   " in vcf".format(variant_id))
                else:
                    posititon_variants.add(variant_id)
            # Check if vcf is sorted
            else:
                if not current_pos >= previous_pos:
                    raise VcfError("Vcf if not sorted in a correct way")
                previous_pos = current_pos
                # Reset posititon_variants since we are on a new position
                posititon_variants = set([variant_id])

    if variant_type != expected_type:
        raise VcfError("VCF file does not only include {0}s, please check vcf {1}".format(
                        expected_type.upper(), vcf_path))

    LOG.info("Vcf file %s looks fine", vcf_path)
    LOG.info("Nr of variants in vcf: {0}".format(nr_variants))
    LOG.info("Type of variants in vcf: {0}".format(variant_type))

    vcf_info = {
        'nr_variants': nr_variants,
        'variant_type': variant_type,
        'individuals': individuals,
    }

    return vcf_info


Instruction:
Explain what the following Python 3 code does
def is_dn(s):
    if s == '':
        return True
    rm = DN_REGEX.match(s)
    return rm is not None and rm.group(0) == s

Response: Return True if s is a LDAP DN.


Instruction:
Implement a Python 3 function for
writing string line as one or more folded lines.

Response:
def _fold_line(self, line):
        """Write string line as one or more folded lines."""
        if len(line) <= self._cols:
            self._output_file.write(line)
            self._output_file.write(self._line_sep)
        else:
            pos = self._cols
            self._output_file.write(line[0:self._cols])
            self._output_file.write(self._line_sep)
            while pos < len(line):
                self._output_file.write(b' ')
                end = min(len(line), pos + self._cols - 1)
                self._output_file.write(line[pos:end])
                self._output_file.write(self._line_sep)
                pos = end


Instruction:
Here you have a function in Python 3, explain what it does
def _needs_base64_encoding(self, attr_type, attr_value):
        return attr_type.lower() in self._base64_attrs or \
            isinstance(attr_value, bytes) or \
            UNSAFE_STRING_RE.search(attr_value) is not None

Response: Return True if attr_value has to be base - 64 encoded.


Instruction:
Can you write a function in Python 3 where it
writes a single attribute type / value pair.

Response:
def _unparse_attr(self, attr_type, attr_value):
        """Write a single attribute type/value pair."""
        if self._needs_base64_encoding(attr_type, attr_value):
            if not isinstance(attr_value, bytes):
                attr_value = attr_value.encode(self._encoding)
            encoded = base64.encodestring(attr_value)\
                .replace(b'\n', b'')\
                .decode('ascii')
            line = ':: '.join([attr_type, encoded])
        else:
            line = ': '.join([attr_type, attr_value])
        self._fold_line(line.encode('ascii'))


Instruction:
Given the following Python 3 function, write the documentation
def _unparse_entry_record(self, entry):
        for attr_type in sorted(entry.keys()):
            for attr_value in entry[attr_type]:
                self._unparse_attr(attr_type, attr_value)

Response: Unparse a record from the database.


Instruction:
How would you explain what the following Python 3 function does
def _unparse_changetype(self, mod_len):
        if mod_len == 2:
            changetype = 'add'
        elif mod_len == 3:
            changetype = 'modify'
        else:
            raise ValueError("modlist item of wrong length")

        self._unparse_attr('changetype', changetype)

Response: Detect and write the changetype.


Instruction:
Can you write a function in Python 3 where it
writes an entry or change record to the output file.

Response:
def unparse(self, dn, record):
        """Write an entry or change record to the output file.

        :type dn: string
        :param dn: distinguished name

        :type record: Union[Dict[string, List[string]], List[Tuple]]
        :param record: Either a dictionary holding  an entry or a list of
            additions (2-tuple) or modifications (3-tuple).
        """
        self._unparse_attr('dn', dn)
        if isinstance(record, dict):
            self._unparse_entry_record(record)
        elif isinstance(record, list):
            self._unparse_change_record(record)
        else:
            raise ValueError("Argument record must be dictionary or list")
        self._output_file.write(self._line_sep)
        self.records_written += 1


Instruction:
Implement a Python 3 function for
stripping trailing line separators from s but no other whitespaces.

Response:
def _strip_line_sep(self, s):
        """Strip trailing line separators from s, but no other whitespaces."""
        if s[-2:] == b'\r\n':
            return s[:-2]
        elif s[-1:] == b'\n':
            return s[:-1]
        else:
            return s


Instruction:
Write a Python 3 function for
iterating input unfoled lines. Skip comments.

Response:
def _iter_unfolded_lines(self):
        """Iter input unfoled lines. Skip comments."""
        line = self._input_file.readline()
        while line:
            self.line_counter += 1
            self.byte_counter += len(line)

            line = self._strip_line_sep(line)

            nextline = self._input_file.readline()
            while nextline and nextline[:1] == b' ':
                line += self._strip_line_sep(nextline)[1:]
                nextline = self._input_file.readline()

            if not line.startswith(b'#'):
                yield line
            line = nextline


Instruction:
Can you create a Python 3 function that
iterates input lines in blocks separated by blank lines.

Response:
def _iter_blocks(self):
        """Iter input lines in blocks separated by blank lines."""
        lines = []
        for line in self._iter_unfolded_lines():
            if line:
                lines.append(line)
            elif lines:
                self.records_read += 1
                yield lines
                lines = []
        if lines:
            self.records_read += 1
            yield lines


Instruction:
Can you generate the documentation for the following Python 3 function
def _parse_attr(self, line):
        colon_pos = line.index(b':')
        attr_type = line[0:colon_pos].decode('ascii')

        if line[colon_pos:].startswith(b'::'):
            attr_value = base64.decodestring(line[colon_pos + 2:])
        elif line[colon_pos:].startswith(b':<'):
            url = line[colon_pos + 2:].strip()
            attr_value = b''
            if self._process_url_schemes:
                u = urlparse(url)
                if u[0] in self._process_url_schemes:
                    attr_value = urlopen(url.decode('ascii')).read()
        else:
            attr_value = line[colon_pos + 1:].strip()

        return self._decode_value(attr_type, attr_value)

Response: Parse a single attribute type and value pair.


Instruction:
Can you tell what is the following Python 3 function doing
def _check_dn(self, dn, attr_value):
        if dn is not None:
            self._error('Two lines starting with dn: in one record.')
        if not is_dn(attr_value):
            self._error('No valid string-representation of '
                'distinguished name %s.' % attr_value)

Response: Check dn attribute for issues.


Instruction:
Create a Python 3 function for
checking changetype attribute for issues.

Response:
def _check_changetype(self, dn, changetype, attr_value):
        """Check changetype attribute for issues."""
        if dn is None:
            self._error('Read changetype: before getting valid dn: line.')
        if changetype is not None:
            self._error('Two lines starting with changetype: in one record.')
        if attr_value not in CHANGE_TYPES:
            self._error('changetype value %s is invalid.' % attr_value)


Instruction:
Can you generate the documentation for the following Python 3 function
def _parse_entry_record(self, lines):
        dn = None
        entry = OrderedDict()

        for line in lines:
            attr_type, attr_value = self._parse_attr(line)

            if attr_type == 'dn':
                self._check_dn(dn, attr_value)
                dn = attr_value
            elif attr_type == 'version' and dn is None:
                pass  # version = 1
            else:
                if dn is None:
                    self._error('First line of record does not start '
                        'with "dn:": %s' % attr_type)
                if attr_value is not None and \
                         attr_type.lower() not in self._ignored_attr_types:
                    if attr_type in entry:
                        entry[attr_type].append(attr_value)
                    else:
                        entry[attr_type] = [attr_value]

        return dn, entry

Response: Parse a single entry record from a list of lines.


Instruction:
Here you have a function in Python 3, explain what it does
def _create_w_objective(m, X, Z=None):
    genes, clusters = m.shape
    cells = X.shape[1]
    nonzeros = (X!=0)
    def objective(w):
        # convert w into a matrix first... because it's a vector for
        # optimization purposes
        w = w.reshape((m.shape[1], X.shape[1]))
        d = m.dot(w)+eps
        # derivative of objective wrt all elements of w
        # for w_{ij}, the derivative is... m_j1+...+m_jn sum over genes minus
        # x_ij
        temp = X/d
        m_sum = m.T.dot(nonzeros)
        m2 = m.T.dot(temp)
        deriv = m_sum - m2
        return np.sum(nonzeros*(d - X*np.log(d)))/genes, deriv.flatten()/genes
    return objective

Response: Create an objective function and its derivative for W given M and X.


Instruction:
Write a Python 3 script to
use a Zero - inflated Poisson Mixture model to estimate cell states and cell mixing weights.

Response:
def zip_estimate_state(data, clusters, init_means=None, init_weights=None, max_iters=10, tol=1e-4, disp=True, inner_max_iters=400, normalize=True):
    """
    Uses a Zero-inflated Poisson Mixture model to estimate cell states and
    cell state mixing weights.

    Args:
        data (array): genes x cells
        clusters (int): number of mixture components
        init_means (array, optional): initial centers - genes x clusters. Default: kmeans++ initializations
        init_weights (array, optional): initial weights - clusters x cells. Default: random(0,1)
        max_iters (int, optional): maximum number of iterations. Default: 10
        tol (float, optional): if both M and W change by less than tol (in RMSE), then the iteration is stopped. Default: 1e-4
        disp (bool, optional): whether or not to display optimization parameters. Default: True
        inner_max_iters (int, optional): Number of iterations to run in the scipy minimizer for M and W. Default: 400
        normalize (bool, optional): True if the resulting W should sum to 1 for each cell. Default: True.

    Returns:
        M: genes x clusters - state centers
        W: clusters x cells - state mixing components for each cell
        ll: final log-likelihood
    """
    genes, cells = data.shape
    # TODO: estimate ZIP parameter?
    if init_means is None:
        means, assignments = kmeans_pp(data, clusters)
    else:
        means = init_means.copy()
    clusters = means.shape[1]
    w_init = np.random.random(cells*clusters)
    if init_weights is not None:
        if len(init_weights.shape)==1:
            init_weights = initialize_from_assignments(init_weights, clusters)
        w_init = init_weights.reshape(cells*clusters)
    m_init = means.reshape(genes*clusters)
    # using zero-inflated parameters...
    L, Z = zip_fit_params_mle(data)
    # repeat steps 1 and 2 until convergence:
    ll = np.inf
    for i in range(max_iters):
        if disp:
            print('iter: {0}'.format(i))
        w_bounds = [(0, 1.0) for x in w_init]
        m_bounds = [(0, None) for x in m_init]
        # step 1: given M, estimate W
        w_objective = _create_w_objective(means, data, Z)
        w_res = minimize(w_objective, w_init, method='L-BFGS-B', jac=True, bounds=w_bounds, options={'disp':disp, 'maxiter':inner_max_iters})
        w_diff = np.sqrt(np.sum((w_res.x-w_init)**2))/w_init.size
        w_new = w_res.x.reshape((clusters, cells))
        w_init = w_res.x
        # step 2: given W, update M
        m_objective = _create_m_objective(w_new, data, Z)
        # method could be 'L-BFGS-B' or 'SLSQP'... SLSQP gives a memory error...
        # or use TNC...
        m_res = minimize(m_objective, m_init, method='L-BFGS-B', jac=True, bounds=m_bounds, options={'disp':disp, 'maxiter':inner_max_iters})
        ll = m_res.fun
        m_diff = np.sqrt(np.sum((m_res.x-m_init)**2))/m_init.size
        m_new = m_res.x.reshape((genes, clusters))
        m_init = m_res.x
        means = m_new
        if w_diff < tol and m_diff < tol:
            break
    if normalize:
        w_new = w_new/w_new.sum(0)
    return m_new, w_new, ll


Instruction:
Write a Python 3 script to
generate k - means ++ initial centers.

Response:
def kmeans_pp(data, k, centers=None):
    """
    Generates kmeans++ initial centers.

    Args:
        data (array): A 2d array- genes x cells
        k (int): Number of clusters
        centers (array, optional): if provided, these are one or more known cluster centers. 2d array of genes x number of centers (<=k).

    Returns:
        centers - a genes x k array of cluster means.
        assignments - a cells x 1 array of cluster assignments
    """
    # TODO: what if there is missing data for a given gene?
    # missing data could be if all the entires are -1.
    genes, cells = data.shape
    if sparse.issparse(data) and not sparse.isspmatrix_csc(data):
        data = sparse.csc_matrix(data)
    num_known_centers = 0
    if centers is None:
        centers = np.zeros((genes, k))
    else:
        num_known_centers = centers.shape[1]
        centers = np.concatenate((centers, np.zeros((genes, k-num_known_centers))), 1)
    distances = np.zeros((cells, k))
    distances[:] = np.inf
    if num_known_centers == 0:
        init = np.random.randint(0, cells)
        if sparse.issparse(data):
            centers[:,0] = data[:, init].toarray().flatten()
        else:
            centers[:,0] = data[:, init]
        num_known_centers+=1
    available_cells = list(range(cells))
    for c in range(num_known_centers, k):
        c2 = c-1
        # use different formulation for distance... if sparse, use lls
        # if not sparse, use poisson_dist
        if sparse.issparse(data):
            lls = poisson_ll(data, centers[:,c2:c2+1]).flatten()
            distances[:,c2] = 1 + lls.max() - lls
            distances[:,c2] /= distances[:,c2].max()
        else:
            for cell in range(cells):
                distances[cell, c2] = poisson_dist(data[:,cell], centers[:,c2])
        # choose a new data point as center... probability proportional
        # to distance^2
        min_distances = np.min(distances, 1)
        min_distances = min_distances**2
        min_distances = min_distances[available_cells]
        # should be sampling without replacement
        min_dist = np.random.choice(available_cells,
                p=min_distances/min_distances.sum())
        available_cells.pop(available_cells.index(min_dist))
        if sparse.issparse(data):
            centers[:,c] = data[:, min_dist].toarray().flatten()
        else:
            centers[:,c] = data[:, min_dist]
    lls = poisson_ll(data, centers)
    new_assignments = np.argmax(lls, 1)
    centers[centers==0.0] = eps
    return centers, new_assignments


Instruction:
Can you generate a brief explanation for the following Python 3 code
def poisson_cluster(data, k, init=None, max_iters=100):
    # TODO: be able to use a combination of fixed and unknown starting points
    # e.g., have init values only for certain genes, have a row of all
    # zeros indicating that kmeans++ should be used for that row.
    genes, cells = data.shape
    #print 'starting: ', centers
    if sparse.issparse(data) and not sparse.isspmatrix_csc(data):
        data = sparse.csc_matrix(data)
    init, assignments = kmeans_pp(data, k, centers=init)
    centers = np.copy(init)
    assignments = np.zeros(cells)
    for it in range(max_iters):
        lls = poisson_ll(data, centers)
        #cluster_dists = np.zeros((cells, k))
        new_assignments = np.argmax(lls, 1)
        if np.equal(assignments, new_assignments).all():
            #print 'ending: ', centers
            return new_assignments, centers
        for c in range(k):
            if sparse.issparse(data):
                if data[:,new_assignments==c].shape[0]==0:
                    # re-initialize centers?
                    new_c, _ = kmeans_pp(data, k, centers[:,:c])
                    centers[:,c] = new_c[:,c]
                else:
                    centers[:,c] = np.asarray(data[:,new_assignments==c].mean(1)).flatten()
            else:
                if len(data[:,new_assignments==c])==0:
                    new_c, _ = kmeans_pp(data, k, centers[:,:c])
                    centers[:,c] = new_c[:,c]
                else:
                    centers[:,c] = np.mean(data[:,new_assignments==c], 1)
        assignments = new_assignments
    return assignments, centers

Response: Performs Poisson hard EM on the given data.


Instruction:
Can you create a Python 3 function that
displays cases in the database.

Response:
def cases(ctx, case_id, to_json):
    """Display cases in the database."""

    adapter = ctx.obj['adapter']
    cases = []

    if case_id:
        case_obj = adapter.case({'case_id':case_id})
        if not case_obj:
            LOG.info("Case {0} does not exist in database".format(case_id))
            return
        case_obj['_id'] = str(case_obj['_id'])
        cases.append(case_obj)
    else:
        cases = adapter.cases()
        if cases.count() == 0:
            LOG.info("No cases found in database")
            ctx.abort()

    if to_json:
        click.echo(json.dumps(cases))
        return

    click.echo("#case_id\tvcf_path")

    for case_obj in cases:
        click.echo("{0}\t{1}".format(case_obj.get('case_id'), case_obj.get('vcf_path')))


Instruction:
Write a Python 3 function for
displaying variants in the database.

Response:
def variants(ctx, variant_id, chromosome, end_chromosome, start, end, variant_type,
             sv_type):
    """Display variants in the database."""
    if sv_type:
        variant_type = 'sv'

    adapter = ctx.obj['adapter']

    if (start or end):
        if not (chromosome and start and end):
            LOG.warning("Regions must be specified with chromosome, start and end")
            return

    if variant_id:
        variant = adapter.get_variant({'_id':variant_id})
        if variant:
            click.echo(variant)
        else:
            LOG.info("Variant {0} does not exist in database".format(variant_id))
        return

    if variant_type == 'snv':
        result = adapter.get_variants(
            chromosome=chromosome,
            start=start,
            end=end
        )
    else:
        LOG.info("Search for svs")
        result = adapter.get_sv_variants(
            chromosome=chromosome,
            end_chromosome=end_chromosome,
            sv_type=sv_type,
            pos=start,
            end=end
        )

    i = 0
    for variant in result:
        i += 1
        pp(variant)

    LOG.info("Number of variants found in database: %s", i)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def dotd(A, B, out=None):
    A = asarray(A, float)
    B = asarray(B, float)
    if A.ndim == 1 and B.ndim == 1:
        if out is None:
            return dot(A, B)
        return dot(A, B, out)
    if out is None:
        out = empty((A.shape[0],), float)
    return einsum("ij,ji->i", A, B, out=out)

Response: r Diagonal of A \ mathrm A \ mathrm B \ intercal.


Instruction:
Can you generate the documentation for the following Python 3 function
def ddot(L, R, left=None, out=None):
    L = asarray(L, float)
    R = asarray(R, float)
    if left is None:
        ok = min(L.ndim, R.ndim) == 1 and max(L.ndim, R.ndim) == 2
        if not ok:
            msg = "Wrong array layout. One array should have"
            msg += " ndim=1 and the other one ndim=2."
            raise ValueError(msg)
        left = L.ndim == 1
    if left:
        if out is None:
            out = copy(R)
        L = L.reshape(list(L.shape) + [1] * (R.ndim - 1))
        return multiply(L, R, out=out)
    else:
        if out is None:
            out = copy(L)
        return multiply(L, R, out=out)

Response: r Dot product of a matrix and a diagonal one.


Instruction:
Explain what the following Python 3 code does
def nanrankdata(a, axis=-1, inplace=False):
    from scipy.stats import rankdata

    if hasattr(a, "dtype") and issubdtype(a.dtype, integer):
        raise ValueError("Integer type is not supported.")

    if isinstance(a, (tuple, list)):
        if inplace:
            raise ValueError("Can't use `inplace=True` for {}.".format(type(a)))
        a = asarray(a, float)

    orig_shape = a.shape
    if a.ndim == 1:
        a = a.reshape(orig_shape + (1,))

    if not inplace:
        a = a.copy()

    def rank1d(x):
        idx = ~isnan(x)
        x[idx] = rankdata(x[idx])
        return x

    a = a.swapaxes(1, axis)
    a = apply_along_axis(rank1d, 0, a)
    a = a.swapaxes(1, axis)

    return a.reshape(orig_shape)

Response: Rank data for arrays contaning NaN values.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def economic_qs(K, epsilon=sqrt(finfo(float).eps)):

    (S, Q) = eigh(K)

    nok = abs(max(Q[0].min(), Q[0].max(), key=abs)) < epsilon
    nok = nok and abs(max(K.min(), K.max(), key=abs)) >= epsilon
    if nok:
        from scipy.linalg import eigh as sp_eigh

        (S, Q) = sp_eigh(K)

    ok = S >= epsilon
    nok = logical_not(ok)
    S0 = S[ok]
    Q0 = Q[:, ok]
    Q1 = Q[:, nok]
    return ((Q0, Q1), S0)

Response: r Economic eigen decomposition for symmetric matrices.


Instruction:
Explain what the following Python 3 code does
def unique(ar):

    import dask.array as da

    if isinstance(ar, da.core.Array):
        return da.unique(ar)

    return _unique(ar)

Response: r Find the unique elements of an array.


Instruction:
Make a summary of the following Python 3 code
def lu_slogdet(LU):
    LU = (asarray(LU[0], float), asarray(LU[1], float))
    adet = _sum(log(_abs(LU[0].diagonal())))

    s = prod(sign(LU[0].diagonal()))

    nrows_exchange = LU[1].size - _sum(LU[1] == arange(LU[1].size, dtype="int32"))

    odd = nrows_exchange % 2 == 1
    if odd:
        s *= -1.0

    return (s, adet)

Response: Natural logarithm of a LU decomposition.


Instruction:
Explain what the following Python 3 code does
def lu_solve(LU, b):
    from scipy.linalg import lu_solve as sp_lu_solve

    LU = (asarray(LU[0], float), asarray(LU[1], float))
    b = asarray(b, float)
    return sp_lu_solve(LU, b, check_finite=False)

Response: r Solve for LU decomposition.


Instruction:
Explain what the following Python 3 code does
def dotd(A, B):
    A = asarray(A, float)
    B = asarray(B, float)
    if A.ndim == 1 and B.ndim == 1:
        return dot(A, B)

    out = empty((A.shape[0],), float)
    out[:] = sum(A * B.T, axis=1)

    return out

Response: r Diagonal of A \ mathrm A \ mathrm B \ intercal.


Instruction:
Can you tell what is the following Python 3 function doing
def economic_svd(G, epsilon=sqrt(finfo(float).eps)):
    from scipy.linalg import svd

    G = asarray(G, float)
    (U, S, V) = svd(G, full_matrices=False, check_finite=False)
    ok = S >= epsilon
    S = S[ok]
    U = U[:, ok]
    V = V[ok, :]
    return (U, S, V)

Response: r Economic Singular Value Decomposition.


Instruction:
Given the following Python 3 function, write the documentation
def rsolve(A, b, epsilon=_epsilon):
    A = asarray(A, float)
    b = asarray(b, float)
    if A.shape[0] == 0:
        return zeros((A.shape[1],))
    if A.shape[1] == 0:
        return zeros((0,))
    try:
        x = lstsq(A, b, rcond=epsilon)
        r = sum(x[3] > epsilon)
        if r == 0:
            return zeros(A.shape[1])
        return x[0]
    except (ValueError, LinAlgError) as e:
        warnings.warn(str(e), RuntimeWarning)
    return solve(A, b)

Response: r Robust solve for the linear equations.


Instruction:
Create a Python 3 function to
check if A is a semi - definite positive matrix.

Response:
def check_semidefinite_positiveness(A):
    """Check if ``A`` is a semi-definite positive matrix.

    Args:
        A (array_like): Matrix.

    Returns:
        bool: ``True`` if ``A`` is definite positive; ``False`` otherwise.
    """
    B = empty_like(A)
    B[:] = A
    B[diag_indices_from(B)] += sqrt(finfo(float).eps)
    try:
        cholesky(B)
    except LinAlgError:
        return False
    return True


Instruction:
Explain what the following Python 3 code does
def check_symmetry(A):
    A = asanyarray(A)
    if A.ndim != 2:
        raise ValueError("Checks symmetry only for bi-dimensional arrays.")

    if A.shape[0] != A.shape[1]:
        return False

    return abs(A - A.T).max() < sqrt(finfo(float).eps)

Response: Checks if A is a symmetric matrix.


Instruction:
Make a summary of the following Python 3 code
def cho_solve(L, b):
    from scipy.linalg import cho_solve as sp_cho_solve

    L = asarray(L, float)
    b = asarray(b, float)
    if L.size == 0:
        if b.size != 0:
            raise ValueError("Dimension mismatch between L and b.")
        return empty(b.shape)
    return sp_cho_solve((L, True), b, check_finite=False)

Response: r Solve for Cholesky decomposition.


Instruction:
Write a Python 3 function that can
get filename and check if in getcwd then get from the package resources folder

Response:
def file_or_resource(fname=None):
    '''get filename and check if in getcwd then get from
    the package resources folder
    '''
    if fname is not None:
        filename = os.path.expanduser(fname)

        resource_package = opentargets_validator.__name__
        resource_path = os.path.sep.join(('resources', filename))

        abs_filename = os.path.join(os.path.abspath(os.getcwd()), filename) \
                       if not os.path.isabs(filename) else filename

        return abs_filename if os.path.isfile(abs_filename) \
            else res.resource_filename(resource_package, resource_path)


Instruction:
Make a summary of the following Python 3 code
def stl(A, b):
    from scipy.linalg import solve_triangular

    A = asarray(A, float)
    b = asarray(b, float)
    return solve_triangular(A, b, lower=True, check_finite=False)

Response: r Solve a lower - triangular matrix A and b.


Instruction:
Can you tell what is the following Python 3 function doing
def _windows_resolve(command):
    try:
        import win32api
    except ImportError:
        if (2, 8) < sys.version_info < (3, 5):
            logger.info(
                "Resolving executable names only supported on Python 2.7 and 3.5+"
            )
        else:
            logger.warning(
                "Could not resolve executable name: package win32api missing"
            )
        return command

    if not command or not isinstance(command[0], six.string_types):
        return command

    try:
        _, found_executable = win32api.FindExecutable(command[0])
        logger.debug("Resolved %s as %s", command[0], found_executable)
        return (found_executable,) + tuple(command[1:])
    except Exception as e:
        if not hasattr(e, "winerror"):
            raise
        # Keep this error message for later in case we fail to resolve the name
        logwarning = getattr(e, "strerror", str(e))

    if "." in command[0]:
        # Special case. The win32api may not properly check file extensions, so
        # try to resolve the executable explicitly.
        for extension in os.getenv("PATHEXT").split(os.pathsep):
            try:
                _, found_executable = win32api.FindExecutable(command[0] + extension)
                logger.debug("Resolved %s as %s", command[0], found_executable)
                return (found_executable,) + tuple(command[1:])
            except Exception as e:
                if not hasattr(e, "winerror"):
                    raise

    logger.warning("Error trying to resolve the executable: %s", logwarning)
    return command

Response: Resolves the executable name and returns the command array with the resolved name.


Instruction:
Can you implement a function in Python 3 that
runs a command line and returns a dictionary containing the output of the command line and the exit code.

Response:
def run(
    command,
    timeout=None,
    debug=False,
    stdin=None,
    print_stdout=True,
    print_stderr=True,
    callback_stdout=None,
    callback_stderr=None,
    environment=None,
    environment_override=None,
    win32resolve=True,
    working_directory=None,
):
    """
    Run an external process.

    File system path objects (PEP-519) are accepted in the command, environment,
    and working directory arguments.

    :param array command: Command line to be run, specified as array.
    :param timeout: Terminate program execution after this many seconds.
    :param boolean debug: Enable further debug messages.
    :param stdin: Optional string that is passed to command stdin.
    :param boolean print_stdout: Pass stdout through to sys.stdout.
    :param boolean print_stderr: Pass stderr through to sys.stderr.
    :param callback_stdout: Optional function which is called for each
                            stdout line.
    :param callback_stderr: Optional function which is called for each
                            stderr line.
    :param dict environment: The full execution environment for the command.
    :param dict environment_override: Change environment variables from the
                                      current values for command execution.
    :param boolean win32resolve: If on Windows, find the appropriate executable
                                 first. This allows running of .bat, .cmd, etc.
                                 files without explicitly specifying their
                                 extension.
    :param string working_directory: If specified, run the executable from
                                     within this working directory.
    :return: A ReturnObject() containing the executed command, stdout and stderr
             (both as bytestrings), and the exitcode. Further values such as
             process runtime can be accessed as dictionary values.
    """

    time_start = time.strftime("%Y-%m-%d %H:%M:%S GMT", time.gmtime())
    logger.debug("Starting external process: %s", command)

    if stdin is None:
        stdin_pipe = None
    else:
        assert sys.platform != "win32", "stdin argument not supported on Windows"
        stdin_pipe = subprocess.PIPE

    start_time = timeit.default_timer()
    if timeout is not None:
        max_time = start_time + timeout

    if environment is not None:
        env = {key: _path_resolve(environment[key]) for key in environment}
    else:
        env = os.environ
    if environment_override:
        env = copy.copy(env)
        env.update(
            {
                key: str(_path_resolve(environment_override[key]))
                for key in environment_override
            }
        )

    command = tuple(_path_resolve(part) for part in command)
    if win32resolve and sys.platform == "win32":
        command = _windows_resolve(command)

    p = subprocess.Popen(
        command,
        shell=False,
        cwd=_path_resolve(working_directory),
        env=env,
        stdin=stdin_pipe,
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
    )

    thread_pipe_pool = []
    notifyee, notifier = Pipe(False)
    thread_pipe_pool.append(notifyee)
    stdout = _NonBlockingStreamReader(
        p.stdout,
        output=print_stdout,
        debug=debug,
        notify=notifier.close,
        callback=callback_stdout,
    )
    notifyee, notifier = Pipe(False)
    thread_pipe_pool.append(notifyee)
    stderr = _NonBlockingStreamReader(
        p.stderr,
        output=print_stderr,
        debug=debug,
        notify=notifier.close,
        callback=callback_stderr,
    )
    if stdin is not None:
        notifyee, notifier = Pipe(False)
        thread_pipe_pool.append(notifyee)
        stdin = _NonBlockingStreamWriter(
            p.stdin, data=stdin, debug=debug, notify=notifier.close
        )

    timeout_encountered = False

    while (p.returncode is None) and (
        (timeout is None) or (timeit.default_timer() < max_time)
    ):
        if debug and timeout is not None:
            logger.debug("still running (T%.2fs)" % (timeit.default_timer() - max_time))

        # wait for some time or until a stream is closed
        try:
            if thread_pipe_pool:
                # Wait for up to 0.5 seconds or for a signal on a remaining stream,
                # which could indicate that the process has terminated.
                try:
                    event = thread_pipe_pool[0].poll(0.5)
                except IOError as e:
                    # on Windows this raises "IOError: [Errno 109] The pipe has been ended"
                    # which is for all intents and purposes equivalent to a True return value.
                    if e.errno != 109:
                        raise
                    event = True
                if event:
                    # One-shot, so remove stream and watch remaining streams
                    thread_pipe_pool.pop(0)
                    if debug:
                        logger.debug("Event received from stream thread")
            else:
                time.sleep(0.5)
        except KeyboardInterrupt:
            p.kill()  # if user pressed Ctrl+C we won't be able to produce a proper report anyway
            # but at least make sure the child process dies with us
            raise

        # check if process is still running
        p.poll()

    if p.returncode is None:
        # timeout condition
        timeout_encountered = True
        if debug:
            logger.debug("timeout (T%.2fs)" % (timeit.default_timer() - max_time))

        # send terminate signal and wait some time for buffers to be read
        p.terminate()
        if thread_pipe_pool:
            thread_pipe_pool[0].poll(0.5)
        if not stdout.has_finished() or not stderr.has_finished():
            time.sleep(2)
        p.poll()

    if p.returncode is None:
        # thread still alive
        # send kill signal and wait some more time for buffers to be read
        p.kill()
        if thread_pipe_pool:
            thread_pipe_pool[0].poll(0.5)
        if not stdout.has_finished() or not stderr.has_finished():
            time.sleep(5)
        p.poll()

    if p.returncode is None:
        raise RuntimeError("Process won't terminate")

    runtime = timeit.default_timer() - start_time
    if timeout is not None:
        logger.debug(
            "Process ended after %.1f seconds with exit code %d (T%.2fs)"
            % (runtime, p.returncode, timeit.default_timer() - max_time)
        )
    else:
        logger.debug(
            "Process ended after %.1f seconds with exit code %d"
            % (runtime, p.returncode)
        )

    stdout = stdout.get_output()
    stderr = stderr.get_output()
    time_end = time.strftime("%Y-%m-%d %H:%M:%S GMT", time.gmtime())

    result = ReturnObject(
        {
            "exitcode": p.returncode,
            "command": command,
            "stdout": stdout,
            "stderr": stderr,
            "timeout": timeout_encountered,
            "runtime": runtime,
            "time_start": time_start,
            "time_end": time_end,
        }
    )
    if stdin is not None:
        result.update(
            {
                "stdin_bytes_sent": stdin.bytes_sent(),
                "stdin_bytes_remain": stdin.bytes_remaining(),
            }
        )

    return result


Instruction:
Can you tell what is the following Python 3 function doing
def run_process_dummy(command, **kwargs):
    warnings.warn(
        "procrunner.run_process_dummy() is deprecated", DeprecationWarning, stacklevel=2
    )

    time_start = time.strftime("%Y-%m-%d %H:%M:%S GMT", time.gmtime())
    logger.info("run_process is disabled. Requested command: %s", command)

    result = ReturnObject(
        {
            "exitcode": 0,
            "command": command,
            "stdout": "",
            "stderr": "",
            "timeout": False,
            "runtime": 0,
            "time_start": time_start,
            "time_end": time_start,
        }
    )
    if kwargs.get("stdin") is not None:
        result.update(
            {"stdin_bytes_sent": len(kwargs["stdin"]), "stdin_bytes_remain": 0}
        )
    return result

Response: A stand - in function that returns a valid result dictionary indicating a successful execution of a command.


Instruction:
Create a Python 3 function for
adding a single character to the buffer.

Response:
def add(self, data):
        """
        Add a single character to buffer. If one or more full lines are found,
        print them (if desired) and pass to callback function.
        """
        data = self._decoder.decode(data)
        if not data:
            return
        self._buffer += data
        if "\n" in data:
            to_print, remainder = self._buffer.rsplit("\n")
            if self._print:
                try:
                    print(to_print)
                except UnicodeEncodeError:
                    print(to_print.encode(sys.getdefaultencoding(), errors="replace"))
                    if not hasattr(self, "_warned"):
                        logger.warning("output encoding error, characters replaced")
                        setattr(self, "_warned", True)
            if self._callback:
                self._callback(to_print)
            self._buffer = remainder


Instruction:
Implement a Python 3 function for
printing and send any remaining data to callback function.

Response:
def flush(self):
        """Print/send any remaining data to callback function."""
        self._buffer += self._decoder.decode(b"", final=True)
        if self._buffer:
            if self._print:
                print(self._buffer)
            if self._callback:
                self._callback(self._buffer)
        self._buffer = ""


Instruction:
Create a Python 3 function for
retrieving the stored data in full.

Response:
def get_output(self):
        """
        Retrieve the stored data in full.
        This call may block if the reading thread has not yet terminated.
        """
        self._closing = True
        if not self.has_finished():
            if self._debug:
                # Main thread overtook stream reading thread.
                underrun_debug_timer = timeit.default_timer()
                logger.warning("NBSR underrun")
            self._thread.join()
            if not self.has_finished():
                if self._debug:
                    logger.debug(
                        "NBSR join after %f seconds, underrun not resolved"
                        % (timeit.default_timer() - underrun_debug_timer)
                    )
                raise Exception("thread did not terminate")
            if self._debug:
                logger.debug(
                    "NBSR underrun resolved after %f seconds"
                    % (timeit.default_timer() - underrun_debug_timer)
                )
        if self._closed:
            raise Exception("streamreader double-closed")
        self._closed = True
        data = self._buffer.getvalue()
        self._buffer.close()
        return data


Instruction:
Write a Python 3 function that can
convert Hiragana to Full - width Katakana.

Response:
def hira2kata(text, ignore=''):
    """Convert Hiragana to Full-width (Zenkaku) Katakana.

    Parameters
    ----------
    text : str
        Hiragana string.
    ignore : str
        Characters to be ignored in converting.

    Return
    ------
    str
        Katakana string.

    Examples
    --------
    >>> print(jaconv.hira2kata('ともえまみ'))
    トモエマミ
    >>> print(jaconv.hira2kata('まどまぎ', ignore='ど'))
    マどマギ
    """
    if ignore:
        h2k_map = _exclude_ignorechar(ignore, H2K_TABLE.copy())
        return _convert(text, h2k_map)
    return _convert(text, H2K_TABLE)


Instruction:
Can you generate the documentation for the following Python 3 function
def hira2hkata(text, ignore=''):
    if ignore:
        h2hk_map = _exclude_ignorechar(ignore, H2HK_TABLE.copy())
        return _convert(text, h2hk_map)
    return _convert(text, H2HK_TABLE)

Response: Convert Hiragana to Hankaku


Instruction:
How would you implement a function in Python 3 that
converts Full - width Katakana to Hiragana.

Response:
def kata2hira(text, ignore=''):
    """Convert Full-width Katakana to Hiragana

    Parameters
    ----------
    text : str
        Full-width Katakana string.
    ignore : str
        Characters to be ignored in converting.

    Return
    ------
    str
        Hiragana string.

    Examples
    --------
    >>> print(jaconv.kata2hira('巴マミ'))
    巴まみ
    >>> print(jaconv.kata2hira('マミサン', ignore='ン'))
    まみさン
    """
    if ignore:
        k2h_map = _exclude_ignorechar(ignore, K2H_TABLE.copy())
        return _convert(text, k2h_map)
    return _convert(text, K2H_TABLE)


Instruction:
Can you implement a function in Python 3 that
converts Hankaku to Zenkaku

Response:
def h2z(text, ignore='', kana=True, ascii=False, digit=False):
    """Convert Half-width (Hankaku) Katakana to Full-width (Zenkaku) Katakana

    Parameters
    ----------
    text : str
        Half-width Katakana string.
    ignore : str
        Characters to be ignored in converting.
    kana : bool
        Either converting Kana or not.
    ascii : bool
        Either converting ascii or not.
    digit : bool
        Either converting digit or not.

    Return
    ------
    str
        Full-width Katakana string.

    Examples
    --------
    >>> print(jaconv.h2z('ﾃｨﾛﾌｨﾅｰﾚ'))
    ティロフィナーレ
    >>> print(jaconv.h2z('ﾃｨﾛﾌｨﾅｰﾚ', ignore='ｨ'))
    テｨロフｨナーレ
    >>> print(jaconv.h2z('abcd', ascii=True))
    ＡＢＣＤ
    >>> print(jaconv.h2z('1234', digit=True))
    １２３４
    """
    def _conv_dakuten(text):
        """Convert Hankaku Dakuten Kana to Zenkaku Dakuten Kana
        """
        text = text.replace("ｶﾞ", "ガ").replace("ｷﾞ", "ギ")
        text = text.replace("ｸﾞ", "グ").replace("ｹﾞ", "ゲ")
        text = text.replace("ｺﾞ", "ゴ").replace("ｻﾞ", "ザ")
        text = text.replace("ｼﾞ", "ジ").replace("ｽﾞ", "ズ")
        text = text.replace("ｾﾞ", "ゼ").replace("ｿﾞ", "ゾ")
        text = text.replace("ﾀﾞ", "ダ").replace("ﾁﾞ", "ヂ")
        text = text.replace("ﾂﾞ", "ヅ").replace("ﾃﾞ", "デ")
        text = text.replace("ﾄﾞ", "ド").replace("ﾊﾞ", "バ")
        text = text.replace("ﾋﾞ", "ビ").replace("ﾌﾞ", "ブ")
        text = text.replace("ﾍﾞ", "ベ").replace("ﾎﾞ", "ボ")
        text = text.replace("ﾊﾟ", "パ").replace("ﾋﾟ", "ピ")
        text = text.replace("ﾌﾟ", "プ").replace("ﾍﾟ", "ペ")
        return text.replace("ﾎﾟ", "ポ").replace("ｳﾞ", "ヴ")

    if ascii:
        if digit:
            if kana:
                h2z_map = H2Z_ALL
            else:
                h2z_map = H2Z_AD
        elif kana:
            h2z_map = H2Z_AK
        else:
            h2z_map = H2Z_A
    elif digit:
        if kana:
            h2z_map = H2Z_DK
        else:
            h2z_map = H2Z_D
    else:
        h2z_map = H2Z_K
    if kana:
        text = _conv_dakuten(text)
    if ignore:
        h2z_map = _exclude_ignorechar(ignore, h2z_map.copy())
    return _convert(text, h2z_map)


Instruction:
Can you generate the documentation for the following Python 3 function
def z2h(text, ignore='', kana=True, ascii=False, digit=False):
    if ascii:
        if digit:
            if kana:
                z2h_map = Z2H_ALL
            else:
                z2h_map = Z2H_AD
        elif kana:
            z2h_map = Z2H_AK
        else:
            z2h_map = Z2H_A
    elif digit:
        if kana:
            z2h_map = Z2H_DK
        else:
            z2h_map = Z2H_D
    else:
        z2h_map = Z2H_K
    if ignore:
        z2h_map = _exclude_ignorechar(ignore, z2h_map.copy())
    return _convert(text, z2h_map)

Response: Convert a string from Zenkaku to Hankaku.


Instruction:
Write a Python 3 script to
convert Hankaku ASCII and DIGIT to Full - width ASCII and DIGIT and Hankaku.

Response:
def normalize(text, mode='NFKC', ignore=''):
    """Convert Half-width (Hankaku) Katakana to Full-width (Zenkaku) Katakana,
    Full-width (Zenkaku) ASCII and DIGIT to Half-width (Hankaku) ASCII
    and DIGIT.
    Additionally, Full-width wave dash (〜) etc. are normalized

    Parameters
    ----------
    text : str
        Source string.
    mode : str
        Unicode normalization mode.
    ignore : str
        Characters to be ignored in converting.

    Return
    ------
    str
        Normalized string.

    Examples
    --------
    >>> print(jaconv.normalize('ﾃｨﾛ･フィナ〜レ', 'NFKC'))
    ティロ・フィナーレ
    """
    text = text.replace('〜', 'ー').replace('～', 'ー')
    text = text.replace("’", "'").replace('”', '"').replace('“', '``')
    text = text.replace('―', '-').replace('‐', '-').replace('˗', '-').replace('֊', '-')
    text = text.replace('‐', '-').replace('‑', '-').replace('‒', '-').replace('–', '-')
    text = text.replace('⁃', '-').replace('⁻', '-').replace('₋', '-').replace('−', '-')
    text = text.replace('﹣', 'ー').replace('－', 'ー').replace('—', 'ー').replace('―', 'ー')
    text = text.replace('━', 'ー').replace('─', 'ー')
    return unicodedata.normalize(mode, text)


Instruction:
Can you implement a function in Python 3 that
converts Hiragana to hepburn - style alphabets.

Response:
def kana2alphabet(text):
    """Convert Hiragana to hepburn-style alphabets

    Parameters
    ----------
    text : str
        Hiragana string.

    Return
    ------
    str
        Hepburn-style alphabets string.

    Examples
    --------
    >>> print(jaconv.kana2alphabet('まみさん'))
    mamisan
    """
    text = text.replace('きゃ', 'kya').replace('きゅ', 'kyu').replace('きょ', 'kyo')
    text = text.replace('ぎゃ', 'gya').replace('ぎゅ', 'gyu').replace('ぎょ', 'gyo')
    text = text.replace('しゃ', 'sha').replace('しゅ', 'shu').replace('しょ', 'sho')
    text = text.replace('じゃ', 'ja').replace('じゅ', 'ju').replace('じょ', 'jo')
    text = text.replace('ちゃ', 'cha').replace('ちゅ', 'chu').replace('ちょ', 'cho')
    text = text.replace('にゃ', 'nya').replace('にゅ', 'nyu').replace('にょ', 'nyo')
    text = text.replace('ふぁ', 'fa').replace('ふぃ', 'fi').replace('ふぇ', 'fe')
    text = text.replace('ふぉ', 'fo')
    text = text.replace('ひゃ', 'hya').replace('ひゅ', 'hyu').replace('ひょ', 'hyo')
    text = text.replace('みゃ', 'mya').replace('みゅ', 'myu').replace('みょ', 'myo')
    text = text.replace('りゃ', 'rya').replace('りゅ', 'ryu').replace('りょ', 'ryo')
    text = text.replace('びゃ', 'bya').replace('びゅ', 'byu').replace('びょ', 'byo')
    text = text.replace('ぴゃ', 'pya').replace('ぴゅ', 'pyu').replace('ぴょ', 'pyo')
    text = text.replace('が', 'ga').replace('ぎ', 'gi').replace('ぐ', 'gu')
    text = text.replace('げ', 'ge').replace('ご', 'go').replace('ざ', 'za')
    text = text.replace('じ', 'ji').replace('ず', 'zu').replace('ぜ', 'ze')
    text = text.replace('ぞ', 'zo').replace('だ', 'da').replace('ぢ', 'ji')
    text = text.replace('づ', 'zu').replace('で', 'de').replace('ど', 'do')
    text = text.replace('ば', 'ba').replace('び', 'bi').replace('ぶ', 'bu')
    text = text.replace('べ', 'be').replace('ぼ', 'bo').replace('ぱ', 'pa')
    text = text.replace('ぴ', 'pi').replace('ぷ', 'pu').replace('ぺ', 'pe')
    text = text.replace('ぽ', 'po')
    text = text.replace('か', 'ka').replace('き', 'ki').replace('く', 'ku')
    text = text.replace('け', 'ke').replace('こ', 'ko').replace('さ', 'sa')
    text = text.replace('し', 'shi').replace('す', 'su').replace('せ', 'se')
    text = text.replace('そ', 'so').replace('た', 'ta').replace('ち', 'chi')
    text = text.replace('つ', 'tsu').replace('て', 'te').replace('と', 'to')
    text = text.replace('な', 'na').replace('に', 'ni').replace('ぬ', 'nu')
    text = text.replace('ね', 'ne').replace('の', 'no').replace('は', 'ha')
    text = text.replace('ひ', 'hi').replace('ふ', 'fu').replace('へ', 'he')
    text = text.replace('ほ', 'ho').replace('ま', 'ma').replace('み', 'mi')
    text = text.replace('む', 'mu').replace('め', 'me').replace('も', 'mo')
    text = text.replace('ら', 'ra').replace('り', 'ri').replace('る', 'ru')
    text = text.replace('れ', 're').replace('ろ', 'ro')
    text = text.replace('や', 'ya').replace('ゆ', 'yu').replace('よ', 'yo')
    text = text.replace('わ', 'wa').replace('ゐ', 'wi').replace('を', 'wo')
    text = text.replace('ゑ', 'we')
    text = _convert(text, KANA2HEP)
    while 'っ' in text:
        text = list(text)
        tsu_pos = text.index('っ')
        if len(text) <= tsu_pos + 1:
            return ''.join(text[:-1]) + 'xtsu'
        if tsu_pos == 0:
            text[tsu_pos] = 'xtsu'
        else:
            text[tsu_pos] = text[tsu_pos + 1]
        text = ''.join(text)
    return text


Instruction:
Implement a function in Python 3 to
convert alphabets to Kiragana.

Response:
def alphabet2kana(text):
    """Convert alphabets to Hiragana

    Parameters
    ----------
    text : str
        Alphabets string.

    Return
    ------
    str
        Hiragana string.

    Examples
    --------
    >>> print(jaconv.alphabet2kana('mamisan'))
    まみさん
    """
    text = text.replace('kya', 'きゃ').replace('kyu', 'きゅ').replace('kyo', 'きょ')
    text = text.replace('gya', 'ぎゃ').replace('gyu', 'ぎゅ').replace('gyo', 'ぎょ')
    text = text.replace('sha', 'しゃ').replace('shu', 'しゅ').replace('sho', 'しょ')
    text = text.replace('zya', 'じゃ').replace('zyu', 'じゅ').replace('zyo', 'じょ')
    text = text.replace('zyi', 'じぃ').replace('zye', 'じぇ')
    text = text.replace('ja', 'じゃ').replace('ju', 'じゅ').replace('jo', 'じょ')
    text = text.replace('jya', 'じゃ').replace('jyu', 'じゅ').replace('jyo', 'じょ')
    text = text.replace('cha', 'ちゃ').replace('chu', 'ちゅ').replace('cho', 'ちょ')
    text = text.replace('tya', 'ちゃ').replace('tyu', 'ちゅ').replace('tyo', 'ちょ')
    text = text.replace('nya', 'にゃ').replace('nyu', 'にゅ').replace('nyo', 'にょ')
    text = text.replace('hya', 'ひゃ').replace('hyu', 'ひゅ').replace('hyo', 'ひょ')
    text = text.replace('mya', 'みゃ').replace('myu', 'みゅ').replace('myo', 'みょ')
    text = text.replace('rya', 'りゃ').replace('ryu', 'りゅ').replace('ryo', 'りょ')
    text = text.replace('bya', 'びゃ').replace('byu', 'びゅ').replace('byo', 'びょ')
    text = text.replace('pya', 'ぴゃ').replace('pyu', 'ぴゅ').replace('pyo', 'ぴょ')
    text = text.replace('oh', 'おお')
    text = text.replace('ga', 'が').replace('gi', 'ぎ').replace('gu', 'ぐ')
    text = text.replace('ge', 'げ').replace('go', 'ご').replace('za', 'ざ')
    text = text.replace('ji', 'じ').replace('zu', 'ず').replace('ze', 'ぜ')
    text = text.replace('zo', 'ぞ').replace('da', 'だ').replace('ji', 'ぢ').replace('di', 'ぢ')
    text = text.replace('va', 'ゔぁ').replace('vi', 'ゔぃ').replace('vu', 'ゔ')
    text = text.replace('ve', 'ゔぇ').replace('vo', 'ゔぉ').replace('vya', 'ゔゃ')
    text = text.replace('vyi', 'ゔぃ').replace('vyu', 'ゔゅ').replace('vye', 'ゔぇ')
    text = text.replace('vyo', 'ゔょ')
    text = text.replace('zu', 'づ').replace('de', 'で').replace('do', 'ど')
    text = text.replace('ba', 'ば').replace('bi', 'び').replace('bu', 'ぶ')
    text = text.replace('be', 'べ').replace('bo', 'ぼ').replace('pa', 'ぱ')
    text = text.replace('pi', 'ぴ').replace('pu', 'ぷ').replace('pe', 'ぺ')
    text = text.replace('po', 'ぽ').replace('dha', 'でゃ').replace('dhi', 'でぃ')
    text = text.replace('dhu', 'でゅ').replace('dhe', 'でぇ').replace('dho', 'でょ')
    text = text.replace('ka', 'か').replace('ki', 'き').replace('ku', 'く')
    text = text.replace('ke', 'け').replace('ko', 'こ').replace('sa', 'さ')
    text = text.replace('shi', 'し').replace('su', 'す').replace('se', 'せ')
    text = text.replace('so', 'そ').replace('ta', 'た').replace('chi', 'ち')
    text = text.replace('tsu', 'つ').replace('te', 'て').replace('to', 'と')
    text = text.replace('na', 'な').replace('ni', 'に').replace('nu', 'ぬ')
    text = text.replace('ne', 'ね').replace('no', 'の').replace('ha', 'は')
    text = text.replace('hi', 'ひ').replace('fu', 'ふ').replace('he', 'へ')
    text = text.replace('ho', 'ほ').replace('ma', 'ま').replace('mi', 'み')
    text = text.replace('mu', 'む').replace('me', 'め').replace('mo', 'も')
    text = text.replace('ra', 'ら').replace('ri', 'り').replace('ru', 'る')
    text = text.replace('re', 'れ').replace('ro', 'ろ')
    text = text.replace('ya', 'や').replace('yu', 'ゆ').replace('yo', 'よ')
    text = text.replace('wa', 'わ').replace('wi', 'ゐ').replace('we', 'ゑ')
    text = text.replace('wo', 'を')
    text = text.replace('nn', 'ん').replace('tu', 'つ').replace('hu', 'ふ')
    text = text.replace('fa', 'ふぁ').replace('fi', 'ふぃ').replace('fe', 'ふぇ')
    text = text.replace('fo', 'ふぉ').replace('-', 'ー')
    text = _convert(text, HEP2KANA)
    ret = []
    for (i, char) in enumerate(text):
        if char in consonants:
            char = 'っ'
        ret.append(char)
    return ''.join(ret)


Instruction:
Make a summary of the following Python 3 code
def histpoints(x, bins=None, xerr=None, yerr='gamma', normed=False, **kwargs):
    import matplotlib.pyplot as plt

    if bins is None:
        bins = calc_nbins(x)

    h, bins = np.histogram(x, bins=bins)
    width = bins[1] - bins[0]
    center = (bins[:-1] + bins[1:]) / 2
    area = sum(h * width)

    if isinstance(yerr, str):
        yerr = poisson_limits(h, yerr)

    if xerr == 'binwidth':
        xerr = width / 2

    if normed:
        h = h / area
        yerr = yerr / area
        area = 1.

    if not 'color' in kwargs:
        kwargs['color'] = 'black'

    if not 'fmt' in kwargs:
        kwargs['fmt'] = 'o'

    plt.errorbar(center, h, xerr=xerr, yerr=yerr, **kwargs)

    return center, (yerr[0], h, yerr[1]), area

Response: Plots a histogram of x using individual data points x and y.


Instruction:
Given the following Python 3 function, write the documentation
def pool(self):
        self._pool = self._pool or eventlet.GreenPool(size=self.pool_size)
        return self._pool

Response: Get an eventlet pool used to dispatch requests.


Instruction:
Explain what the following Python 3 code does
def start(self):
        if self.pid is not None:

            LOG.error(
                "The process is already running with pid {0}.".format(self.pid)
            )
            sys.exit(exit.ALREADY_RUNNING)

        self.daemonize()

        LOG.info("Beginning run loop for process.")
        try:

            self.run()

        except Exception:

            LOG.exception("Uncaught exception in the daemon run() method.")
            self.stop()
            sys.exit(exit.RUN_FAILURE)

Response: Start the process with daemonization.


Instruction:
Given the following Python 3 function, write the documentation
def stop(self):
        if self.pid is None:

            return None

        try:

            while True:

                self.send(signal.SIGTERM)
                time.sleep(0.1)

        except RuntimeError as err:

            if "No such process" in str(err):

                LOG.info("Succesfully stopped the process.")
                return None

            LOG.exception("Failed to stop the process:")
            sys.exit(exit.STOP_FAILED)

        except TypeError as err:

            if "an integer is required" in str(err):

                LOG.info("Succesfully stopped the process.")
                return None

            LOG.exception("Failed to stop the process:")
            sys.exit(exit.STOP_FAILED)

Response: Stop the daemonized process.


Instruction:
Here you have a function in Python 3, explain what it does
def handle(self, signum, handler):
        if not isinstance(signum, int):

            raise TypeError(
                "Signals must be given as integers. Got {0}.".format(
                    type(signum),
                ),
            )

        if not callable(handler):

            raise TypeError(
                "Signal handlers must be callable.",
            )

        signal.signal(signum, self._handle_signals)
        self._handlers[signum].append(handler)

Response: Set a function to run when the given signal is recieved.


Instruction:
How would you implement a function in Python 3 that
sends the given signal to the running process.

Response:
def send(self, signum):
        """Send the given signal to the running process.

        If the process is not running a RuntimeError with a message of "No such
        process" should be emitted.
        """
        if not isinstance(signum, int):

            raise TypeError(
                "Signals must be given as integers. Got {0}.".format(
                    type(signum),
                ),
            )

        try:

            os.kill(self.pid, signum)

        except OSError as err:

            if "No such process" in err.strerror:

                raise RuntimeError("No such process {0}.".format(self.pid))

            raise err


Instruction:
How would you implement a function in Python 3 that
handles all signals which trigger a process stop.

Response:
def shutdown(self, signum):
        """Handle all signals which trigger a process stop.

        This method should run all appropriate signal handlers registered
        through the 'handle' method. At the end it should cause the process
        to exit with a status code. If any of the handlers raise an exception
        the exit code should be SHUTDOWN_FAILED otherwise SUCCESS.
        """
        dirty = False
        for handler in self._handlers[signum]:

            try:

                handler()

            except:

                LOG.exception("A shutdown handler failed to execute:")
                dirty = True

        del self.pid

        if dirty:

            sys.exit(exit.SHUTDOWN_FAILED)
            return None

        sys.exit(exit.SUCCESS)
        return None


Instruction:
Can you generate the documentation for the following Python 3 function
def preprocess_path(path):
    if path.startswith("gs://"):
        fs = gcsfs.GCSFileSystem(token=get_gcloud_token())
        for sub_f in fs.ls(path):
            if sub_f.endswith("/") and sub_f.split("/")[-2] == FILES_FOLDER:
                return "gs://" + sub_f
        return path
    if path.startswith("hdfs://"):
        return path
    for sub_f in os.listdir(path):
        sub_f_tot = os.path.join(path, sub_f)
        if os.path.isdir(sub_f_tot) and sub_f == FILES_FOLDER:
            if check_for_dataset(sub_f_tot):
                return sub_f_tot
            else:
                raise ValueError("Dataset in {} was not in GMQL format".format(sub_f_tot))
    # if we are here it means that there is no files folder...so we need to check the root
    if check_for_dataset(path):
        return path
    else:
        raise ValueError("Dataset in {} was not in GMQL format".format(path))

Response: This function takes a path and checks if the file is in the GMQL format.


Instruction:
Here you have a function in Python 3, explain what it does
def check_for_dataset(files):
    all_files = os.listdir(files)
    meta_files = set(map(lambda y: y[: -9], filter(lambda x: x.endswith(".gdm.meta"), all_files)))
    regs_files = set(map(lambda y: y[: -4], filter(lambda x: x.endswith(".gdm"), all_files)))
    return meta_files == regs_files

Response: Check if the path of the dataset contains a gdm dataset.


Instruction:
Given the following Python 3 function, write the documentation
def load_from_path(local_path=None, parser=None,  all_load=False):

    from .. import GDataframe
    from .. import GMQLDataset
    pmg = get_python_manager()

    local_path = preprocess_path(local_path)

    if all_load:
        # load directly the metadata for exploration
        meta = MetaLoaderFile.load_meta_from_path(local_path)
        if isinstance(parser, RegionParser):
            # region data
            regs = RegLoaderFile.load_reg_from_path(local_path, parser)
        else:
            regs = RegLoaderFile.load_reg_from_path(local_path)

        return GDataframe.GDataframe(regs=regs, meta=meta)
    else:
        from ...settings import is_metaprofiling_enabled
        if is_metaprofiling_enabled():
            meta_profile = create_metadata_profile(local_path)
        else:
            meta_profile = None

        if parser is None:
            # find the parser
            parser = RegLoaderFile.get_parser(local_path)
        elif not isinstance(parser, RegionParser):
            raise ValueError("parser must be RegionParser. {} was provided".format(type(parser)))

        source_table = get_source_table()
        id = source_table.search_source(local=local_path)
        if id is None:
            id = source_table.add_source(local=local_path, parser=parser)
        local_sources = [id]

        index = pmg.read_dataset(str(id), parser.get_gmql_parser())
        return GMQLDataset.GMQLDataset(index=index, parser=parser,
                                       location="local", path_or_name=local_path,
                                       local_sources=local_sources,
                                       meta_profile=meta_profile)

Response: Loads the data from a local path into a GMQLDataset.


Instruction:
Implement a Python 3 function for
loading the data from a remote repository.

Response:
def load_from_remote(remote_name, owner=None):
    """ Loads the data from a remote repository.

    :param remote_name: The name of the dataset in the remote repository
    :param owner: (optional) The owner of the dataset. If nothing is provided, the current user
                  is used. For public datasets use 'public'.

    :return: A new GMQLDataset or a GDataframe
    """
    from .. import GMQLDataset
    pmg = get_python_manager()
    remote_manager = get_remote_manager()
    parser = remote_manager.get_dataset_schema(remote_name, owner)

    source_table = get_source_table()
    id = source_table.search_source(remote=remote_name)
    if id is None:
        id = source_table.add_source(remote=remote_name, parser=parser)
    index = pmg.read_dataset(str(id), parser.get_gmql_parser())
    remote_sources = [id]
    return GMQLDataset.GMQLDataset(index=index, location="remote", path_or_name=remote_name,
                                   remote_sources=remote_sources)