Instruction:
Create a Python 3 function for
populating the report from the xml xml

Response:
def process(self):
        """
        populate the report from the xml
        :return:
        """
        suites = None
        if isinstance(self.tree, ET.Element):
            root = self.tree
        else:
            root = self.tree.getroot()

        if root.tag == "testrun":
            root = root[0]

        if root.tag == "testsuite":
            suites = [root]

        if root.tag == "testsuites":
            suites = [x for x in root]

        assert suites, "could not find test suites in results xml"

        for suite in suites:
            cursuite = Suite()
            self.suites.append(cursuite)
            cursuite.name = suite.attrib["name"]
            if "package" in suite.attrib:
                cursuite.package = suite.attrib["package"]
            cursuite.duration = float(suite.attrib.get("time", '0').replace(',',''))

            for element in suite:
                if element.tag == "error":
                    # top level error?
                    errtag = {
                        "message": element.attrib.get("message", ""),
                        "type": element.attrib.get("type", ""),
                        "text": element.text
                    }
                    cursuite.errors.append(errtag)
                if element.tag == "system-out":
                    cursuite.stdout = element.text
                if element.tag == "system-err":
                    cursuite.stderr = element.text

                if element.tag == "properties":
                    for prop in element:
                        if prop.tag == "property":
                            cursuite.properties[prop.attrib["name"]] = prop.attrib["value"]

                if element.tag == "testcase":
                    testcase = element

                    if not testcase.attrib.get("classname", None):
                        testcase.attrib["classname"] = NO_CLASSNAME

                    if testcase.attrib["classname"] not in cursuite:
                        testclass = Class()
                        testclass.name = testcase.attrib["classname"]
                        cursuite[testclass.name] = testclass

                    testclass = cursuite[testcase.attrib["classname"]]
                    newcase = Case()
                    newcase.name = testcase.attrib["name"]
                    newcase.testclass = testclass
                    newcase.duration = float(testcase.attrib.get("time", '0').replace(',',''))
                    testclass.cases.append(newcase)

                    # does this test case have any children?
                    for child in testcase:
                        if child.tag == "skipped":
                            newcase.skipped = child.text
                            if "message" in child.attrib:
                                newcase.skipped_msg = child.attrib["message"]
                        elif child.tag == "system-out":
                            newcase.stdout = child.text
                        elif child.tag == "system-err":
                            newcase.stderr = child.text
                        elif child.tag == "failure":
                            newcase.failure = child.text
                            if "message" in child.attrib:
                                newcase.failure_msg = child.attrib["message"]
                        elif child.tag == "error":
                            newcase.failure = child.text
                            if "message" in child.attrib:
                                newcase.failure_msg = child.attrib["message"]
                        elif child.tag == "properties":
                            for property in child:
                                newproperty = Property()
                                newproperty.name = property.attrib["name"]
                                newproperty.value = property.attrib["value"]
                                newcase.properties.append(newproperty)


Instruction:
Can you implement a function in Python 3 that
makes a table of contents listing each suite

Response:
def toc(self):
        """
        If this report has multiple suite results, make a table of contents listing each suite
        :return:
        """
        if len(self.suites) > 1:
            tochtml = "
"
            for suite in self.suites:
                tochtml += '
{name}
'.format(
                        anchor=suite.anchor(),
                        name=tag.text(suite.name))
            tochtml += "
"
            return tochtml
        else:
            return ""


Instruction:
Can you implement a function in Python 3 that
renders the test suite as an HTML report with links to errors first.

Response:
def html(self):
        """
        Render the test suite as a HTML report with links to errors first.
        :return:
        """

        page = self.get_html_head()
        page += "<body><h1>Test Report</h1>"
        page += self.toc()
        for suite in self.suites:
            page += suite.html()
        page += "</body></html>"

        return page


Instruction:
Can you tell what is the following Python 3 function doing
def run(args):
    (opts, args) = PARSER.parse_args(args) if args else PARSER.parse_args()
    if not len(args):
        PARSER.print_usage()
        sys.exit(1)

    outfilename = args[0] + ".html"
    if len(args) > 1:
        outfilename = args[1]

    report = parser.Junit(args[0])
    html = report.html()

    with open(outfilename, "wb") as outfile:
        outfile.write(html.encode('utf-8'))

Response:
Run this tool
    :param args:
    :return:


Instruction:
Can you implement a function in Python 3 that
generates a series of remarks that the user should consider when interpreting the summary statistics.

Response:
def _generate_remark_str(self, end_of_line = '\n'):
        """
        Generate a series of remarks that the user should consider
        when interpreting the summary statistics.
        """
        warnings = {}
        msg = '{}'.format(end_of_line)

        # generate warnings for continuous variables
        if self._continuous:
            # highlight far outliers
            outlier_mask = self.cont_describe.far_outliers > 1
            outlier_vars = list(self.cont_describe.far_outliers[outlier_mask].dropna(how='all').index)
            if outlier_vars:
                warnings["Warning, Tukey test indicates far outliers in"] = outlier_vars

            # highlight possible multimodal distributions using hartigan's dip test
            # -1 values indicate NaN
            modal_mask = (self.cont_describe.diptest >= 0) & (self.cont_describe.diptest <= 0.05)
            modal_vars = list(self.cont_describe.diptest[modal_mask].dropna(how='all').index)
            if modal_vars:
                warnings["Warning, Hartigan's Dip Test reports possible multimodal distributions for"] = modal_vars

            # highlight non normal distributions
            # -1 values indicate NaN
            modal_mask = (self.cont_describe.normaltest >= 0) & (self.cont_describe.normaltest <= 0.001)
            modal_vars = list(self.cont_describe.normaltest[modal_mask].dropna(how='all').index)
            if modal_vars:
                warnings["Warning, test for normality reports non-normal distributions for"] = modal_vars

        # create the warning string
        for n,k in enumerate(sorted(warnings)):
            msg += '[{}] {}: {}.{}'.format(n+1,k,', '.join(warnings[k]), end_of_line)

        return msg


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _detect_categorical_columns(self,data):
        # assume all non-numerical and date columns are categorical
        numeric_cols = set(data._get_numeric_data().columns.values)
        date_cols = set(data.select_dtypes(include=[np.datetime64]).columns)
        likely_cat = set(data.columns) - numeric_cols
        likely_cat = list(likely_cat - date_cols)
        # check proportion of unique values if numerical
        for var in data._get_numeric_data().columns:
            likely_flag = 1.0 * data[var].nunique()/data[var].count() < 0.05
            if likely_flag:
                 likely_cat.append(var)
        return likely_cat

Response: Detect categorical columns if they are not specified.


Instruction:
Explain what the following Python 3 code does
def _std(self,x):
        return np.nanstd(x.values,ddof=self._ddof)

Response: Compute standard deviation of a set of nan values.


Instruction:
Given the following Python 3 function, write the documentation
def _tukey(self,x,threshold):
        vals = x.values[~np.isnan(x.values)]
        try:
            q1, q3 = np.percentile(vals, [25, 75])
            iqr = q3 - q1
            low_bound = q1 - (iqr * threshold)
            high_bound = q3 + (iqr * threshold)
            outliers = np.where((vals > high_bound) | (vals < low_bound))
        except:
            outliers = []
        return outliers

Response: Count outliers according to Tukey s rule.


Instruction:
Create a Python 3 function to
compute the number of outliers in a given time series

Response:
def _outliers(self,x):
        """
        Compute number of outliers
        """
        outliers = self._tukey(x, threshold = 1.5)
        return np.size(outliers)


Instruction:
Can you implement a function in Python 3 that
computes the number of far out outliers for a given time series

Response:
def _far_outliers(self,x):
        """
        Compute number of "far out" outliers
        """
        outliers = self._tukey(x, threshold = 3.0)
        return np.size(outliers)


Instruction:
How would you code a function in Python 3 to
compute the summary of the IQR - level logarithmic state of the input series.

Response:
def _t1_summary(self,x):
        """
        Compute median [IQR] or mean (Std) for the input series.

        Parameters
        ----------
            x : pandas Series
                Series of values to be summarised.
        """
        # set decimal places
        if isinstance(self._decimals,int):
            n = self._decimals
        elif isinstance(self._decimals,dict):
            try:
                n = self._decimals[x.name]
            except:
                n = 1
        else:
            n = 1
            warnings.warn('The decimals arg must be an int or dict. Defaulting to {} d.p.'.format(n))

        if x.name in self._nonnormal:
            f = '{{:.{}f}} [{{:.{}f}},{{:.{}f}}]'.format(n,n,n)
            return f.format(np.nanmedian(x.values),
                np.nanpercentile(x.values,25), np.nanpercentile(x.values,75))
        else:
            f = '{{:.{}f}} ({{:.{}f}})'.format(n,n)
            return f.format(np.nanmean(x.values),
                np.nanstd(x.values,ddof=self._ddof))


Instruction:
How would you implement a function in Python 3 that
creates the DataFrame that describe the continuous data.

Response:
def _create_cont_describe(self,data):
        """
        Describe the continuous data.

        Parameters
        ----------
            data : pandas DataFrame
                The input dataset.

        Returns
        ----------
            df_cont : pandas DataFrame
                Summarise the continuous variables.
        """
        aggfuncs = [pd.Series.count,np.mean,np.median,self._std,
            self._q25,self._q75,min,max,self._t1_summary,self._diptest,
            self._outliers,self._far_outliers,self._normaltest]

        # coerce continuous data to numeric
        cont_data = data[self._continuous].apply(pd.to_numeric, errors='coerce')
        # check all data in each continuous column is numeric
        bad_cols = cont_data.count() != data[self._continuous].count()
        bad_cols = cont_data.columns[bad_cols]
        if len(bad_cols)>0:
            raise InputError("""The following continuous column(s) have non-numeric values: {}.
            Either specify the column(s) as categorical or remove the non-numeric values.""".format(bad_cols.values))

        # check for coerced column containing all NaN to warn user
        for column in cont_data.columns[cont_data.count() == 0]:
            self._non_continuous_warning(column)

        if self._groupby:
            # add the groupby column back
            cont_data = cont_data.merge(data[[self._groupby]],
                left_index=True, right_index=True)

            # group and aggregate data
            df_cont = pd.pivot_table(cont_data,
                columns=[self._groupby],
                aggfunc=aggfuncs)
        else:
            # if no groupby, just add single group column
            df_cont = cont_data.apply(aggfuncs).T
            df_cont.columns.name = 'overall'
            df_cont.columns = pd.MultiIndex.from_product([df_cont.columns,
                ['overall']])

        df_cont.index.rename('variable',inplace=True)

        # remove prefix underscore from column names (e.g. _std -> std)
        agg_rename = df_cont.columns.levels[0]
        agg_rename = [x[1:] if x[0]=='_' else x for x in agg_rename]
        df_cont.columns.set_levels(agg_rename, level=0, inplace=True)

        return df_cont


Instruction:
Can you tell what is the following Python 3 function doing
def _create_cat_describe(self,data):
        group_dict = {}

        for g in self._groupbylvls:
            if self._groupby:
                d_slice = data.loc[data[self._groupby] == g, self._categorical]
            else:
                d_slice = data[self._categorical].copy()

            # create a dataframe with freq, proportion
            df = d_slice.copy()

            # convert type to string to avoid int converted to boolean, avoiding nans
            for column in df.columns:
                df[column] = [str(row) if not pd.isnull(row) else None for row in df[column].values]

            df = df.melt().groupby(['variable','value']).size().to_frame(name='freq')
            df.index.set_names('level', level=1, inplace=True)
            df['percent'] = df['freq'].div(df.freq.sum(level=0),level=0).astype(float)* 100

            # set number of decimal places for percent
            if isinstance(self._decimals,int):
                n = self._decimals
                f = '{{:.{}f}}'.format(n)
                df['percent'] = df['percent'].astype(float).map(f.format)
            elif isinstance(self._decimals,dict):
                df.loc[:,'percent'] = df.apply(self._format_cat, axis=1)
            else:
                n = 1
                f = '{{:.{}f}}'.format(n)
                df['percent'] = df['percent'].astype(float).map(f.format)

            # add n column, listing total non-null values for each variable
            ct = d_slice.count().to_frame(name='n')
            ct.index.name = 'variable'
            df = df.join(ct)

            # add null count
            nulls = d_slice.isnull().sum().to_frame(name='isnull')
            nulls.index.name = 'variable'
            # only save null count to the first category for each variable
            # do this by extracting the first category from the df row index
            levels = df.reset_index()[['variable','level']].groupby('variable').first()
            # add this category to the nulls table
            nulls = nulls.join(levels)
            nulls.set_index('level', append=True, inplace=True)
            # join nulls to categorical
            df = df.join(nulls)

            # add summary column
            df['t1_summary'] = df.freq.map(str) + ' (' + df.percent.map(str) + ')'

            # add to dictionary
            group_dict[g] = df

        df_cat = pd.concat(group_dict,axis=1)
        # ensure the groups are the 2nd level of the column index
        if df_cat.columns.nlevels>1:
            df_cat = df_cat.swaplevel(0, 1, axis=1).sort_index(axis=1,level=0)

        return df_cat

Response: Create a dataframe that describe the categorical data.


Instruction:
Create a Python 3 function for
creating a table containing p - values for significance tests.

Response:
def _create_significance_table(self,data):
        """
        Create a table containing p-values for significance tests. Add features of
        the distributions and the p-values to the dataframe.

        Parameters
        ----------
            data : pandas DataFrame
                The input dataset.

        Returns
        ----------
            df : pandas DataFrame
                A table containing the p-values, test name, etc.
        """
        # list features of the variable e.g. matched, paired, n_expected
        df=pd.DataFrame(index=self._continuous+self._categorical,
            columns=['continuous','nonnormal','min_observed','pval','ptest'])

        df.index.rename('variable', inplace=True)
        df['continuous'] = np.where(df.index.isin(self._continuous),True,False)
        df['nonnormal'] = np.where(df.index.isin(self._nonnormal),True,False)

        # list values for each variable, grouped by groupby levels
        for v in df.index:
            is_continuous = df.loc[v]['continuous']
            is_categorical = ~df.loc[v]['continuous']
            is_normal = ~df.loc[v]['nonnormal']

            # if continuous, group data into list of lists
            if is_continuous:
                catlevels = None
                grouped_data = []
                for s in self._groupbylvls:
                    lvl_data = data.loc[data[self._groupby]==s, v]
                    # coerce to numeric and drop non-numeric data
                    lvl_data = lvl_data.apply(pd.to_numeric, errors='coerce').dropna()
                    # append to overall group data
                    grouped_data.append(lvl_data.values)
                min_observed = len(min(grouped_data,key=len))
            # if categorical, create contingency table
            elif is_categorical:
                catlevels = sorted(data[v].astype('category').cat.categories)
                grouped_data = pd.crosstab(data[self._groupby].rename('_groupby_var_'),data[v])
                min_observed = grouped_data.sum(axis=1).min()

            # minimum number of observations across all levels
            df.loc[v,'min_observed'] = min_observed

            # compute pvalues
            df.loc[v,'pval'],df.loc[v,'ptest'] = self._p_test(v,
                grouped_data,is_continuous,is_categorical,
                is_normal,min_observed,catlevels)

        return df


Instruction:
How would you explain what the following Python 3 function does
def _create_cont_table(self,data):
        # remove the t1_summary level
        table = self.cont_describe[['t1_summary']].copy()
        table.columns = table.columns.droplevel(level=0)

        # add a column of null counts as 1-count() from previous function
        nulltable = data[self._continuous].isnull().sum().to_frame(name='isnull')
        try:
            table = table.join(nulltable)
        except TypeError: # if columns form a CategoricalIndex, need to convert to string first
            table.columns = table.columns.astype(str)
            table = table.join(nulltable)

        # add an empty level column, for joining with cat table
        table['level'] = ''
        table.set_index([table.index,'level'],inplace=True)

        # add pval column
        if self._pval and self._pval_adjust:
            table = table.join(self._significance_table[['pval (adjusted)','ptest']])
        elif self._pval:
            table = table.join(self._significance_table[['pval','ptest']])

        return table

Response: Create tableone for continuous data.


Instruction:
Can you tell what is the following Python 3 function doing
def _create_cat_table(self,data):
        table = self.cat_describe['t1_summary'].copy()
        # add the total count of null values across all levels
        isnull = data[self._categorical].isnull().sum().to_frame(name='isnull')
        isnull.index.rename('variable', inplace=True)
        try:
            table = table.join(isnull)
        except TypeError: # if columns form a CategoricalIndex, need to convert to string first
            table.columns = table.columns.astype(str)
            table = table.join(isnull)

        # add pval column
        if self._pval and self._pval_adjust:
            table = table.join(self._significance_table[['pval (adjusted)','ptest']])
        elif self._pval:
            table = table.join(self._significance_table[['pval','ptest']])

        return table

Response: Create table one for categorical data.


Instruction:
Given the following Python 3 function, write the documentation
def _create_tableone(self,data):
        if self._continuous and self._categorical:

            # support pandas<=0.22
            try:
                table = pd.concat([self.cont_table,self.cat_table],sort=False)
            except:
                table = pd.concat([self.cont_table,self.cat_table])
        elif self._continuous:
            table = self.cont_table
        elif self._categorical:
            table = self.cat_table

        # round pval column and convert to string
        if self._pval and self._pval_adjust:
            table['pval (adjusted)'] = table['pval (adjusted)'].apply('{:.3f}'.format).astype(str)
            table.loc[table['pval (adjusted)'] == '0.000', 'pval (adjusted)'] = '<0.001'
        elif self._pval:
            table['pval'] = table['pval'].apply('{:.3f}'.format).astype(str)
            table.loc[table['pval'] == '0.000', 'pval'] = '<0.001'

        # sort the table rows
        table.reset_index().set_index(['variable','level'], inplace=True)
        if self._sort:
            # alphabetical
            new_index = sorted(table.index.values)
        else:
            # sort by the columns argument
            new_index = sorted(table.index.values,key=lambda x: self._columns.index(x[0]))
        table = table.reindex(new_index)

        # if a limit has been set on the number of categorical variables
        # then re-order the variables by frequency
        if self._limit:
            levelcounts = data[self._categorical].nunique()
            levelcounts = levelcounts[levelcounts >= self._limit]
            for v,_ in levelcounts.iteritems():
                count = data[v].value_counts().sort_values(ascending=False)
                new_index = [(v, i) for i in count.index]
                # restructure to match orig_index
                new_index_array=np.empty((len(new_index),), dtype=object)
                new_index_array[:]=[tuple(i) for i in new_index]
                orig_index = table.index.values.copy()
                orig_index[table.index.get_loc(v)] = new_index_array
                table = table.reindex(orig_index)

        # inserts n row
        n_row = pd.DataFrame(columns = ['variable','level','isnull'])
        n_row.set_index(['variable','level'], inplace=True)
        n_row.loc['n', ''] = None

        # support pandas<=0.22
        try:
            table = pd.concat([n_row,table],sort=False)
        except:
            table = pd.concat([n_row,table])

        if self._groupbylvls == ['overall']:
            table.loc['n','overall'] = len(data.index)
        else:
            for g in self._groupbylvls:
                ct = data[self._groupby][data[self._groupby]==g].count()
                table.loc['n',g] = ct

        # only display data in first level row
        dupe_mask = table.groupby(level=[0]).cumcount().ne(0)
        dupe_columns = ['isnull']
        optional_columns = ['pval','pval (adjusted)','ptest']
        for col in optional_columns:
            if col in table.columns.values:
                dupe_columns.append(col)

        table[dupe_columns] = table[dupe_columns].mask(dupe_mask).fillna('')

        # remove empty column added above
        table.drop([''], axis=1, inplace=True)

        # remove isnull column if not needed
        if not self._isnull:
            table.drop('isnull',axis=1,inplace=True)

        # replace nans with empty strings
        table.fillna('',inplace=True)

        # add column index
        if not self._groupbylvls == ['overall']:
            # rename groupby variable if requested
            c = self._groupby
            if self._alt_labels:
                if self._groupby in self._alt_labels:
                    c = self._alt_labels[self._groupby]

            c = 'Grouped by {}'.format(c)
            table.columns = pd.MultiIndex.from_product([[c], table.columns])

        # display alternative labels if assigned
        table.rename(index=self._create_row_labels(), inplace=True, level=0)

        # if a limit has been set on the number of categorical variables
        # limit the number of categorical variables that are displayed
        if self._limit:
            table = table.groupby('variable').head(self._limit)

        # re-order the columns in a consistent fashion
        if self._groupby:
            cols = table.columns.levels[1].values
        else:
            cols = table.columns.values

        if 'isnull' in cols:
            cols = ['isnull'] + [x for x in cols if x != 'isnull']

        # iterate through each optional column
        # if they exist, put them at the end of the dataframe
        # ensures the last 3 columns will be in the same order as optional_columns
        for col in optional_columns:
            if col in cols:
                cols = [x for x in cols if x != col] + [col]

        if self._groupby:
            table = table.reindex(cols, axis=1, level=1)
        else:
            table = table.reindex(cols, axis=1)

        return table

Response: Create a table 1 by combining continuous and categorical tables.


Instruction:
Can you generate the documentation for the following Python 3 function
def _create_row_labels(self):
        # start with the original column names
        labels = {}
        for c in self._columns:
            labels[c] = c

        # replace column names with alternative names if provided
        if self._alt_labels:
            for k in self._alt_labels.keys():
                labels[k] = self._alt_labels[k]

        # append the label suffix
        if self._label_suffix:
            for k in labels.keys():
                if k in self._nonnormal:
                    labels[k] = "{}, {}".format(labels[k],"median [Q1,Q3]")
                elif k in self._categorical:
                    labels[k] = "{}, {}".format(labels[k],"n (%)")
                else:
                    labels[k] = "{}, {}".format(labels[k],"mean (SD)")

        return labels

Response: Create the original labels for rows.


Instruction:
How would you explain what the following Python 3 function does
def dip_pval_tabinterpol(dip, N):
    '''
        dip     -   dip value computed from dip_from_cdf
        N       -   number of observations
    '''

    # if qDiptab_df is None:
    #     raise DataError("Tabulated p-values not available. See installation instructions.")

    if np.isnan(N) or N < 10:
        return np.nan

    qDiptab_dict = {'0': {4: 0.125,
      5: 0.1,
      6: 0.0833333333333333,
      7: 0.0714285714285714,
      8: 0.0625,
      9: 0.0555555555555556,
      10: 0.05,
      15: 0.0341378172277919,
      20: 0.033718563622065004,
      30: 0.0262674485075642,
      50: 0.0218544781364545,
      100: 0.0164852597438403,
      200: 0.0111236388849688,
      500: 0.007554885975761959,
      1000: 0.00541658127872122,
      2000: 0.0039043999745055702,
      5000: 0.00245657785440433,
      10000: 0.00174954269199566,
      20000: 0.00119458814106091,
      40000: 0.000852415648011777,
      72000: 0.000644400053256997},
     '0.01': {4: 0.125,
      5: 0.1,
      6: 0.0833333333333333,
      7: 0.0714285714285714,
      8: 0.0625,
      9: 0.0613018090298924,
      10: 0.0610132555623269,
      15: 0.0546284208048975,
      20: 0.0474333740698401,
      30: 0.0395871890405749,
      50: 0.0314400501999916,
      100: 0.022831985803043,
      200: 0.0165017735429825,
      500: 0.0106403461127515,
      1000: 0.0076028674530018705,
      2000: 0.0054166418179658294,
      5000: 0.0034480928223332603,
      10000: 0.00244595133885302,
      20000: 0.00173435346896287,
      40000: 0.00122883479310665,
      72000: 0.000916872204484283},
     '0.02': {4: 0.125,
      5: 0.1,
      6: 0.0833333333333333,
      7: 0.0714285714285714,
      8: 0.0656911994503283,
      9: 0.0658615858179315,
      10: 0.0651627333214016,
      15: 0.0572191260231815,
      20: 0.0490891387627092,
      30: 0.0414574606741673,
      50: 0.0329008160470834,
      100: 0.0238917486442849,
      200: 0.0172594157992489,
      500: 0.0111255573208294,
      1000: 0.00794987834644799,
      2000: 0.0056617138625232296,
      5000: 0.00360473943713036,
      10000: 0.00255710802275612,
      20000: 0.0018119443458468102,
      40000: 0.0012846930445701802,
      72000: 0.0009579329467655321},
     '0.05': {4: 0.125,
      5: 0.1,
      6: 0.0833333333333333,
      7: 0.0725717816250742,
      8: 0.0738651136071762,
      9: 0.0732651142535317,
      10: 0.0718321619656165,
      15: 0.0610087367689692,
      20: 0.052719998201553,
      30: 0.0444462614069956,
      50: 0.0353023819040016,
      100: 0.0256559537977579,
      200: 0.0185259426032926,
      500: 0.0119353655328931,
      1000: 0.0085216518343594,
      2000: 0.00607120971135229,
      5000: 0.0038632654801084897,
      10000: 0.00273990955227265,
      20000: 0.00194259470485893,
      40000: 0.0013761765052555301,
      72000: 0.00102641863872347},
     '0.1': {4: 0.125,
      5: 0.1,
      6: 0.0833333333333333,
      7: 0.0817315478539489,
      8: 0.0820045917762512,
      9: 0.0803941629593475,
      10: 0.077966212182459,
      15: 0.0642657137330444,
      20: 0.0567795509056742,
      30: 0.0473998525042686,
      50: 0.0377279973102482,
      100: 0.0273987414570948,
      200: 0.0197917612637521,
      500: 0.0127411306411808,
      1000: 0.00909775605533253,
      2000: 0.0064762535755248,
      5000: 0.00412089506752692,
      10000: 0.0029225480567908,
      20000: 0.00207173719623868,
      40000: 0.0014675150200632301,
      72000: 0.0010949515421800199},
     '0.2': {4: 0.125,
      5: 0.1,
      6: 0.0924514470941933,
      7: 0.0940590181922527,
      8: 0.0922700601131892,
      9: 0.0890432420913848,
      10: 0.0852835359834564,
      15: 0.0692234107989591,
      20: 0.0620134674468181,
      30: 0.0516677370374349,
      50: 0.0410699984399582,
      100: 0.0298109370830153,
      200: 0.0215233745778454,
      500: 0.0138524542751814,
      1000: 0.00988924521014078,
      2000: 0.00703573098590029,
      5000: 0.00447640050137479,
      10000: 0.00317374638422465,
      20000: 0.00224993202086955,
      40000: 0.00159376453672466,
      72000: 0.00118904090369415},
     '0.3': {4: 0.125,
      5: 0.1,
      6: 0.103913431059949,
      7: 0.10324449080087102,
      8: 0.0996737189599363,
      9: 0.0950811420297928,
      10: 0.0903204173707099,
      15: 0.0745462114365167,
      20: 0.0660163872069048,
      30: 0.0551037519001622,
      50: 0.0437704598622665,
      100: 0.0317771496530253,
      200: 0.0229259769870428,
      500: 0.0147536004288476,
      1000: 0.0105309297090482,
      2000: 0.007494212545892991,
      5000: 0.00476555693102276,
      10000: 0.00338072258533527,
      20000: 0.00239520831473419,
      40000: 0.00169668445506151,
      72000: 0.00126575197699874},
     '0.4': {4: 0.125,
      5: 0.10872059357632902,
      6: 0.113885220640212,
      7: 0.110964599995697,
      8: 0.10573353180273701,
      9: 0.0999380897811046,
      10: 0.0943334983745117,
      15: 0.0792030878981762,
      20: 0.0696506075066401,
      30: 0.058265005347492994,
      50: 0.0462925642671299,
      100: 0.0336073821590387,
      200: 0.024243848341112,
      500: 0.0155963185751048,
      1000: 0.0111322726797384,
      2000: 0.007920878896017329,
      5000: 0.005037040297500721,
      10000: 0.0035724387653598205,
      20000: 0.00253036792824665,
      40000: 0.0017925341833790601,
      72000: 0.00133750966361506},
     '0.5': {4: 0.125,
      5: 0.12156379802641401,
      6: 0.123071187137781,
      7: 0.11780784650433501,
      8: 0.11103512984770501,
      9: 0.10415356007586801,
      10: 0.0977817630384725,
      15: 0.083621033469191,
      20: 0.0733437740592714,
      30: 0.0614510857304343,
      50: 0.048851155289608,
      100: 0.0354621760592113,
      200: 0.025584358256487003,
      500: 0.0164519238025286,
      1000: 0.0117439009052552,
      2000: 0.008355737247680059,
      5000: 0.0053123924740821294,
      10000: 0.00376734715752209,
      20000: 0.00266863168718114,
      40000: 0.00189061261635977,
      72000: 0.00141049709228472},
     '0.6': {4: 0.125,
      5: 0.134318918697053,
      6: 0.13186973390253,
      7: 0.124216086833531,
      8: 0.11592005574998801,
      9: 0.10800780236193198,
      10: 0.102180866696628,
      15: 0.0881198482202905,
      20: 0.0776460662880254,
      30: 0.0649164408053978,
      50: 0.0516145897865757,
      100: 0.0374805844550272,
      200: 0.0270252129816288,
      500: 0.017383057902553,
      1000: 0.012405033293814,
      2000: 0.00882439333812351,
      5000: 0.00560929919359959,
      10000: 0.00397885007249132,
      20000: 0.0028181999035216,
      40000: 0.00199645471886179,
      72000: 0.00148936709298802},
     '0.7': {4: 0.13255954878268902,
      5: 0.14729879897625198,
      6: 0.140564796497941,
      7: 0.130409013968317,
      8: 0.120561479262465,
      9: 0.112512617124951,
      10: 0.10996094814295099,
      15: 0.093124666680253,
      20: 0.0824558407118372,
      30: 0.0689178762425442,
      50: 0.0548121932066019,
      100: 0.0398046179116599,
      200: 0.0286920262150517,
      500: 0.0184503949887735,
      1000: 0.0131684179320803,
      2000: 0.009367858207170609,
      5000: 0.00595352728377949,
      10000: 0.00422430013176233,
      20000: 0.00299137548142077,
      40000: 0.00211929748381704,
      72000: 0.00158027541945626},
     '0.8': {4: 0.15749736904023498,
      5: 0.161085025702604,
      6: 0.14941924112913002,
      7: 0.136639642123068,
      8: 0.125558759034845,
      9: 0.12291503348081699,
      10: 0.11884476721158699,
      15: 0.0996694393390689,
      20: 0.08834462700173701,
      30: 0.0739249074078291,
      50: 0.0588230482851366,
      100: 0.0427283846799166,
      200: 0.0308006766341406,
      500: 0.0198162679782071,
      1000: 0.0141377942603047,
      2000: 0.01005604603884,
      5000: 0.00639092280563517,
      10000: 0.00453437508148542,
      20000: 0.00321024899920135,
      40000: 0.0022745769870358102,
      72000: 0.00169651643860074},
     '0.9': {4: 0.18740187880755899,
      5: 0.176811998476076,
      6: 0.159137064572627,
      7: 0.144240669035124,
      8: 0.141841067033899,
      9: 0.136412639387084,
      10: 0.130462149644819,
      15: 0.11008749690090598,
      20: 0.0972346018122903,
      30: 0.0814791379390127,
      50: 0.0649136324046767,
      100: 0.047152783315718,
      200: 0.0339967814293504,
      500: 0.0218781313182203,
      1000: 0.0156148055023058,
      2000: 0.0111019116837591,
      5000: 0.00705566126234625,
      10000: 0.00500178808402368,
      20000: 0.00354362220314155,
      40000: 0.00250999080890397,
      72000: 0.0018730618472582602},
     '0.95': {4: 0.20726978858735998,
      5: 0.18639179602794398,
      6: 0.164769608513302,
      7: 0.159903395678336,
      8: 0.153978303998561,
      9: 0.14660378495401902,
      10: 0.139611395137099,
      15: 0.118760769203664,
      20: 0.105130218270636,
      30: 0.0881689143126666,
      50: 0.0702737877191269,
      100: 0.0511279442868827,
      200: 0.0368418413878307,
      500: 0.0237294742633411,
      1000: 0.0169343970067564,
      2000: 0.0120380990328341,
      5000: 0.0076506368153935,
      10000: 0.00542372242836395,
      20000: 0.00384330190244679,
      40000: 0.00272375073486223,
      72000: 0.00203178401610555},
     '0.98': {4: 0.22375580462922195,
      5: 0.19361253363045,
      6: 0.17917654739278197,
      7: 0.17519655327122302,
      8: 0.16597856724751,
      9: 0.157084065653166,
      10: 0.150961728882481,
      15: 0.128890475210055,
      20: 0.11430970428125302,
      30: 0.0960564383013644,
      50: 0.0767095886079179,
      100: 0.0558022052195208,
      200: 0.0402729850316397,
      500: 0.025919578977657003,
      1000: 0.018513067368104,
      2000: 0.0131721010552576,
      5000: 0.00836821687047215,
      10000: 0.00592656681022859,
      20000: 0.00420258799378253,
      40000: 0.00298072958568387,
      72000: 0.00222356097506054},
     '0.99': {4: 0.231796258864192,
      5: 0.19650913979884502,
      6: 0.191862827995563,
      7: 0.184118659121501,
      8: 0.172988528276759,
      9: 0.164164643657217,
      10: 0.159684158858235,
      15: 0.13598356863636,
      20: 0.120624043335821,
      30: 0.101478558893837,
      50: 0.0811998415355918,
      100: 0.059024132304226,
      200: 0.0426864799777448,
      500: 0.0274518022761997,
      1000: 0.0196080260483234,
      2000: 0.0139655122281969,
      5000: 0.00886357892854914,
      10000: 0.00628034732880374,
      20000: 0.00445774902155711,
      40000: 0.00315942194040388,
      72000: 0.00235782814777627},
     '0.995': {4: 0.23726374382677898,
      5: 0.198159967287576,
      6: 0.20210197104296804,
      7: 0.19101439617430602,
      8: 0.179010413496374,
      9: 0.172821674582338,
      10: 0.16719524735674,
      15: 0.14245248368127697,
      20: 0.126552378036739,
      30: 0.10650487144103,
      50: 0.0852854646662134,
      100: 0.0620425065165146,
      200: 0.044958959158761,
      500: 0.0288986369564301,
      1000: 0.0206489568587364,
      2000: 0.0146889122204488,
      5000: 0.00934162787186159,
      10000: 0.00661030641550873,
      20000: 0.00469461513212743,
      40000: 0.0033273652798148,
      72000: 0.00248343580127067},
     '0.998': {4: 0.241992892688593,
      5: 0.19924427936243302,
      6: 0.213015781111186,
      7: 0.198216795232182,
      8: 0.186504388711178,
      9: 0.182555283567818,
      10: 0.175419540856082,
      15: 0.15017281653074202,
      20: 0.13360135382395,
      30: 0.112724636524262,
      50: 0.0904847827490294,
      100: 0.0658016011466099,
      200: 0.0477643873749449,
      500: 0.0306813505050163,
      1000: 0.0219285176765082,
      2000: 0.0156076779647454,
      5000: 0.009932186363240291,
      10000: 0.00702254699967648,
      20000: 0.004994160691291679,
      40000: 0.00353988965698579,
      72000: 0.00264210826339498},
     '0.999': {4: 0.244369839049632,
      5: 0.199617527406166,
      6: 0.219518627282415,
      7: 0.20234101074826102,
      8: 0.19448404115794,
      9: 0.188658833121906,
      10: 0.180611195797351,
      15: 0.15545613369632802,
      20: 0.138569903791767,
      30: 0.117164140184417,
      50: 0.0940930106666244,
      100: 0.0684479731118028,
      200: 0.0497198001867437,
      500: 0.0320170996823189,
      1000: 0.0228689168972669,
      2000: 0.0162685615996248,
      5000: 0.0103498795291629,
      10000: 0.0073182262815645795,
      20000: 0.00520917757743218,
      40000: 0.00369400045486625,
      72000: 0.0027524322157581},
     '0.9995': {4: 0.245966625504691,
      5: 0.19980094149902802,
      6: 0.22433904739444602,
      7: 0.205377566346832,
      8: 0.200864297005026,
      9: 0.19408912076824603,
      10: 0.18528641605039603,
      15: 0.160896499106958,
      20: 0.14336916123968,
      30: 0.12142585990898701,
      50: 0.0974904344916743,
      100: 0.0709169443994193,
      200: 0.0516114611801451,
      500: 0.0332452747332959,
      1000: 0.023738710122235003,
      2000: 0.0168874937789415,
      5000: 0.0107780907076862,
      10000: 0.0076065423418208,
      20000: 0.005403962359243721,
      40000: 0.00383345715372182,
      72000: 0.0028608570740143},
     '0.9998': {4: 0.24743959723326198,
      5: 0.19991708183427104,
      6: 0.22944933215424101,
      7: 0.208306562526874,
      8: 0.20884999705022897,
      9: 0.19915700809389003,
      10: 0.19120308390504398,
      15: 0.16697940794624802,
      20: 0.148940116394883,
      30: 0.126733051889401,
      50: 0.10228420428399698,
      100: 0.0741183486081263,
      200: 0.0540543978864652,
      500: 0.0348335698576168,
      1000: 0.0248334158891432,
      2000: 0.0176505093388153,
      5000: 0.0113184316868283,
      10000: 0.00795640367207482,
      20000: 0.00564540201704594,
      40000: 0.0040079346963469605,
      72000: 0.00298695044508003},
     '0.9999': {4: 0.24823065965663801,
      5: 0.19995902909307503,
      6: 0.232714530449602,
      7: 0.209866047852379,
      8: 0.212556040406219,
      9: 0.20288159843655804,
      10: 0.19580515933918397,
      15: 0.17111793515551002,
      20: 0.152832538183622,
      30: 0.131198578897542,
      50: 0.104680624334611,
      100: 0.0762579402903838,
      200: 0.0558704526182638,
      500: 0.0359832389317461,
      1000: 0.0256126573433596,
      2000: 0.0181944265400504,
      5000: 0.0117329446468571,
      10000: 0.0082270524584354,
      20000: 0.00580460792299214,
      40000: 0.00414892737222885,
      72000: 0.00309340092038059},
     '0.99995': {4: 0.248754269146416,
      5: 0.19997839537608197,
      6: 0.236548128358969,
      7: 0.21096757693345103,
      8: 0.21714917413729898,
      9: 0.205979795735129,
      10: 0.20029398089673,
      15: 0.17590050570443203,
      20: 0.15601016361897102,
      30: 0.133691739483444,
      50: 0.107496694235039,
      100: 0.0785735967934979,
      200: 0.0573877056330228,
      500: 0.0369051995840645,
      1000: 0.0265491336936829,
      2000: 0.0186226037818523,
      5000: 0.0119995948968375,
      10000: 0.00852240989786251,
      20000: 0.00599774739593151,
      40000: 0.0042839159079761,
      72000: 0.00319932767198801},
     '0.99998': {4: 0.24930203997425898,
      5: 0.199993151405815,
      6: 0.2390887911995,
      7: 0.212233348558702,
      8: 0.22170007640450304,
      9: 0.21054115498898,
      10: 0.20565108964621898,
      15: 0.18185667601316602,
      20: 0.16131922583934502,
      30: 0.137831637950694,
      50: 0.11140887547015,
      100: 0.0813458356889133,
      200: 0.0593365901653878,
      500: 0.0387221159256424,
      1000: 0.027578430100535997,
      2000: 0.0193001796565433,
      5000: 0.0124410052027886,
      10000: 0.00892863905540303,
      20000: 0.00633099254378114,
      40000: 0.0044187010443287895,
      72000: 0.00332688234611187},
     '0.99999': {4: 0.24945965232322498,
      5: 0.199995525025673,
      6: 0.24010356643629502,
      7: 0.21266103831250602,
      8: 0.225000835357532,
      9: 0.21180033095039003,
      10: 0.209682048785853,
      15: 0.185743454151004,
      20: 0.165568255916749,
      30: 0.14155750962435099,
      50: 0.113536607717411,
      100: 0.0832963013755522,
      200: 0.0607646310473911,
      500: 0.039930259057650005,
      1000: 0.0284430733108,
      2000: 0.0196241518040617,
      5000: 0.0129467396733128,
      10000: 0.009138539330002129,
      20000: 0.00656987109386762,
      40000: 0.00450818604569179,
      72000: 0.00339316094477355},
     '1': {4: 0.24974836247845,
      5: 0.199999835639211,
      6: 0.24467288361776798,
      7: 0.21353618608817,
      8: 0.23377291968768302,
      9: 0.21537991431762502,
      10: 0.221530282182963,
      15: 0.19224056333056197,
      20: 0.175834459522789,
      30: 0.163833046059817,
      50: 0.11788671686531199,
      100: 0.0926780423096737,
      200: 0.0705309107882395,
      500: 0.0431448163617178,
      1000: 0.0313640941982108,
      2000: 0.0213081254074584,
      5000: 0.014396063834027,
      10000: 0.00952234579566773,
      20000: 0.006858294480462271,
      40000: 0.00513477467565583,
      72000: 0.00376331697005859}}

    qDiptab_df = pd.DataFrame(qDiptab_dict)

    diptable = np.array(qDiptab_df)
    ps = np.array(qDiptab_df.columns).astype(float)
    Ns = np.array(qDiptab_df.index)

    if N >= Ns[-1]:
        dip = transform_dip_to_other_nbr_pts(dip, N, Ns[-1]-0.1)
        N = Ns[-1]-0.1

    iNlow = np.nonzero(Ns < N)[0][-1]
    qN = (N-Ns[iNlow])/(Ns[iNlow+1]-Ns[iNlow])
    dip_sqrtN = np.sqrt(N)*dip
    dip_interpol_sqrtN = (
        np.sqrt(Ns[iNlow])*diptable[iNlow, :] + qN*(
            np.sqrt(Ns[iNlow+1])*diptable[iNlow+1, :]-np.sqrt(Ns[iNlow])*diptable[iNlow, :]))

    if not (dip_interpol_sqrtN < dip_sqrtN).any():
        return 1

    iplow = np.nonzero(dip_interpol_sqrtN < dip_sqrtN)[0][-1]
    if iplow == len(dip_interpol_sqrtN) - 1:
        return 0

    qp = (dip_sqrtN-dip_interpol_sqrtN[iplow])/(dip_interpol_sqrtN[iplow+1]-dip_interpol_sqrtN[iplow])
    p_interpol = ps[iplow] + qp*(ps[iplow+1]-ps[iplow])

    return 1 - p_interpol

Response: Tabulated p - values for a given set of observations.


Instruction:
Make a summary of the following Python 3 code
def dip_and_closest_unimodal_from_cdf(xF, yF, plotting=False, verbose=False, eps=1e-12):
    '''
        Dip computed as distance between empirical distribution function (EDF) and
        cumulative distribution function for the unimodal distribution with
        smallest such distance. The optimal unimodal distribution is found by
        the algorithm presented in

            Hartigan (1985): Computation of the dip statistic to test for
            unimodaliy. Applied Statistics, vol. 34, no. 3

        If the plotting option is enabled the optimal unimodal distribution
        function is plotted along with (xF, yF-dip) and (xF, yF+dip)

        xF  -   x-coordinates for EDF
        yF  -   y-coordinates for EDF

    '''

    ## TODO! Preprocess xF and yF so that yF increasing and xF does
    ## not have more than two copies of each x-value.

    if (xF[1:]-xF[:-1] < -eps).any():
        raise ValueError('Need sorted x-values to compute dip')
    if (yF[1:]-yF[:-1] < -eps).any():
        raise ValueError('Need sorted y-values to compute dip')

    # if plotting:
    #     Nplot = 5
    #     bfig = plt.figure(figsize=(12, 3))
    #     i = 1  # plot index

    D = 0  # lower bound for dip*2

    # [L, U] is interval where we still need to find unimodal function,
    # the modal interval
    L = 0
    U = len(xF) - 1

    # iGfin are the indices of xF where the optimal unimodal distribution is greatest
    # convex minorant to (xF, yF+dip)
    # iHfin are the indices of xF where the optimal unimodal distribution is least
    # concave majorant to (xF, yF-dip)
    iGfin = L
    iHfin = U

    while 1:

        iGG = greatest_convex_minorant_sorted(xF[L:(U+1)], yF[L:(U+1)])
        iHH = least_concave_majorant_sorted(xF[L:(U+1)], yF[L:(U+1)])
        iG = np.arange(L, U+1)[iGG]
        iH = np.arange(L, U+1)[iHH]

        # Interpolate. First and last point are in both and does not need
        # interpolation. Might cause trouble if included due to possiblity
        # of infinity slope at beginning or end of interval.
        if iG[0] != iH[0] or iG[-1] != iH[-1]:
            raise ValueError('Convex minorant and concave majorant should start and end at same points.')
        hipl = np.interp(xF[iG[1:-1]], xF[iH], yF[iH])
        gipl = np.interp(xF[iH[1:-1]], xF[iG], yF[iG])
        hipl = np.hstack([yF[iH[0]], hipl, yF[iH[-1]]])
        gipl = np.hstack([yF[iG[0]], gipl, yF[iG[-1]]])
        #hipl = lin_interpol_sorted(xF[iG], xF[iH], yF[iH])
        #gipl = lin_interpol_sorted(xF[iH], xF[iG], yF[iG])

        # Find largest difference between GCM and LCM.
        gdiff = hipl - yF[iG]
        hdiff = yF[iH] - gipl
        imaxdiffg = np.argmax(gdiff)
        imaxdiffh = np.argmax(hdiff)
        d = max(gdiff[imaxdiffg], hdiff[imaxdiffh])

        # # Plot current GCM and LCM.
        # if plotting:
        #     if i > Nplot:
        #         bfig = plt.figure(figsize=(12, 3))
        #         i = 1
        #     bax = bfig.add_subplot(1, Nplot, i)
        #     bax.plot(xF, yF, color='red')
        #     bax.plot(xF, yF-d/2, color='black')
        #     bax.plot(xF, yF+d/2, color='black')
        #     bax.plot(xF[iG], yF[iG]+d/2, color='blue')
        #     bax.plot(xF[iH], yF[iH]-d/2, color='blue')

        # if d <= D:
        #     if verbose:
        #         print("Difference in modal interval smaller than current dip")
        #     break

        # Find new modal interval so that largest difference is at endpoint
        # and set d to largest distance between current GCM and LCM.
        if gdiff[imaxdiffg] > hdiff[imaxdiffh]:
            L0 = iG[imaxdiffg]
            U0 = iH[iH >= L0][0]
        else:
            U0 = iH[imaxdiffh]
            L0 = iG[iG <= U0][-1]
        # Add points outside the modal interval to the final GCM and LCM.
        iGfin = np.hstack([iGfin, iG[(iG <= L0)*(iG > L)]])
        iHfin = np.hstack([iH[(iH >= U0)*(iH < U)], iHfin])

        # # Plot new modal interval
        # if plotting:
        #     ymin, ymax = bax.get_ylim()
        #     bax.axvline(xF[L0], ymin, ymax, color='orange')
        #     bax.axvline(xF[U0], ymin, ymax, color='red')
        #     bax.set_xlim(xF[L]-.1*(xF[U]-xF[L]), xF[U]+.1*(xF[U]-xF[L]))

        # Compute new lower bound for dip*2
        # i.e. largest difference outside modal interval
        gipl = np.interp(xF[L:(L0+1)], xF[iG], yF[iG])
        D = max(D, np.amax(yF[L:(L0+1)] - gipl))
        hipl = np.interp(xF[U0:(U+1)], xF[iH], yF[iH])
        D = max(D, np.amax(hipl - yF[U0:(U+1)]))

        if xF[U0]-xF[L0] < eps:
            if verbose:
                print("Modal interval zero length")
            break

        # if plotting:
        #     mxpt = np.argmax(yF[L:(L0+1)] - gipl)
        #     bax.plot([xF[L:][mxpt], xF[L:][mxpt]], [yF[L:][mxpt]+d/2,
        #       gipl[mxpt]+d/2], '+', color='red')
        #     mxpt = np.argmax(hipl - yF[U0:(U+1)])
        #     bax.plot([xF[U0:][mxpt], xF[U0:][mxpt]], [yF[U0:][mxpt]-d/2,
        #       hipl[mxpt]-d/2], '+', color='red')
        #     i += 1

        # Change modal interval
        L = L0
        U = U0

        if d <= D:
            if verbose:
                print("Difference in modal interval smaller than new dip")
            break

    # if plotting:

    #     # Add modal interval to figure
    #     bax.axvline(xF[L0], ymin, ymax, color='green', linestyle='dashed')
    #     bax.axvline(xF[U0], ymin, ymax, color='green', linestyle='dashed')

    #     ## Plot unimodal function (not distribution function)
    #     bfig = plt.figure()
    #     bax = bfig.add_subplot(1, 1, 1)
    #     bax.plot(xF, yF, color='red')
    #     bax.plot(xF, yF-D/2, color='black')
    #     bax.plot(xF, yF+D/2, color='black')

    # Find string position in modal interval
    iM = np.arange(iGfin[-1], iHfin[0]+1)
    yM_lower = yF[iM]-D/2
    yM_lower[0] = yF[iM[0]]+D/2
    iMM_concave = least_concave_majorant_sorted(xF[iM], yM_lower)
    iM_concave = iM[iMM_concave]
    #bax.plot(xF[iM], yM_lower, color='orange')
    #bax.plot(xF[iM_concave], yM_lower[iMM_concave], color='red')
    lcm_ipl = np.interp(xF[iM], xF[iM_concave], yM_lower[iMM_concave])
    try:
        mode = iM[np.nonzero(lcm_ipl > yF[iM]+D/2)[0][-1]]
        #bax.axvline(xF[mode], color='green', linestyle='dashed')
    except IndexError:
        iM_convex = np.zeros(0, dtype='i')
    else:
        after_mode = iM_concave > mode
        iM_concave = iM_concave[after_mode]
        iMM_concave = iMM_concave[after_mode]
        iM = iM[iM <= mode]
        iM_convex = iM[greatest_convex_minorant_sorted(xF[iM], yF[iM])]

    # if plotting:

    #     bax.plot(xF[np.hstack([iGfin, iM_convex, iM_concave, iHfin])],
    #              np.hstack([yF[iGfin] + D/2, yF[iM_convex] + D/2,
    #                         yM_lower[iMM_concave], yF[iHfin] - D/2]), color='blue')
    #     #bax.plot(xF[iM], yM_lower, color='orange')

    #     ## Plot unimodal distribution function
    #     bfig = plt.figure()
    #     bax = bfig.add_subplot(1, 1, 1)
    #     bax.plot(xF, yF, color='red')
    #     bax.plot(xF, yF-D/2, color='black')
    #     bax.plot(xF, yF+D/2, color='black')

    # Find string position in modal interval
    iM = np.arange(iGfin[-1], iHfin[0]+1)
    yM_lower = yF[iM]-D/2
    yM_lower[0] = yF[iM[0]]+D/2
    iMM_concave = least_concave_majorant_sorted(xF[iM], yM_lower)
    iM_concave = iM[iMM_concave]
    #bax.plot(xF[iM], yM_lower, color='orange')
    #bax.plot(xF[iM_concave], yM_lower[iMM_concave], color='red')
    lcm_ipl = np.interp(xF[iM], xF[iM_concave], yM_lower[iMM_concave])
    try:
        mode = iM[np.nonzero(lcm_ipl > yF[iM]+D/2)[0][-1]]
        #bax.axvline(xF[mode], color='green', linestyle='dashed')
    except IndexError:
        iM_convex = np.zeros(0, dtype='i')
    else:
        after_mode = iM_concave > mode
        iM_concave = iM_concave[after_mode]
        iMM_concave = iMM_concave[after_mode]
        iM = iM[iM <= mode]
        iM_convex = iM[greatest_convex_minorant_sorted(xF[iM], yF[iM])]

    # Closest unimodal curve
    xU = xF[np.hstack([iGfin[:-1], iM_convex, iM_concave, iHfin[1:]])]
    yU = np.hstack([yF[iGfin[:-1]] + D/2, yF[iM_convex] + D/2,
                    yM_lower[iMM_concave], yF[iHfin[1:]] - D/2])
    # Add points so unimodal curve goes from 0 to 1
    k_start = (yU[1]-yU[0])/(xU[1]-xU[0]+1e-5)
    xU_start = xU[0] - yU[0]/(k_start+1e-5)
    k_end = (yU[-1]-yU[-2])/(xU[-1]-xU[-2]+1e-5)
    xU_end = xU[-1] + (1-yU[-1])/(k_end+1e-5)
    xU = np.hstack([xU_start, xU, xU_end])
    yU = np.hstack([0, yU, 1])

    # if plotting:
    #     bax.plot(xU, yU, color='blue')
    #     #bax.plot(xF[iM], yM_lower, color='orange')
    #     plt.show()

    return D/2, (xU, yU)

Response: Compute the dip and closest unimodal from the original empirical distribution function.


Instruction:
Implement a Python 3 function for
returning the factor for one - dimensional plug - in bandwidth selection.

Response:
def bandwidth_factor(nbr_data_pts, deriv_order=0):
    '''
        Scale factor for one-dimensional plug-in bandwidth selection.
    '''
    if deriv_order == 0:
        return (3.0*nbr_data_pts/4)**(-1.0/5)

    if deriv_order == 2:
        return (7.0*nbr_data_pts/4)**(-1.0/9)

    raise ValueError('Not implemented for derivative of order {}'.format(deriv_order))


Instruction:
How would you explain what the following Python 3 function does
def make_html_tag(tag, text=None, **params):
    params_string = ""

    # Parameters are passed. Turn the dict into a string like "a=1 b=2 c=3" string.
    for key, value in sorted(params.items()):
        # Strip off a leading underscore from the attribute's key to allow attributes like '_class'
        # to be used as a CSS class specification instead of the reserved Python keyword 'class'.
        key = key.lstrip("_")

        params_string += u' {0}="{1}"'.format(key, value)

    # Create the tag string
    tag_string = u"<{0}{1}>".format(tag, params_string)

    # Add text and closing tag if required.
    if text:
        tag_string += u"{0}</{1}>".format(text, tag)

    return tag_string

Response: Create an HTML tag string.


Instruction:
Create a Python 3 function for
returning a string that can be used to render a pager of the current page.

Response:
def pager(
        self,
        format="~2~",
        url=None,
        show_if_single_page=False,
        separator=" ",
        symbol_first="<<",
        symbol_last=">>",
        symbol_previous="<",
        symbol_next=">",
        link_attr=None,
        curpage_attr=None,
        dotdot_attr=None,
        link_tag=None,
    ):
        """
        Return string with links to other pages (e.g. '1 .. 5 6 7 [8] 9 10 11 .. 50').

        format:
            Format string that defines how the pager is rendered. The string
            can contain the following $-tokens that are substituted by the
            string.Template module:

            - $first_page: number of first reachable page
            - $last_page: number of last reachable page
            - $page: number of currently selected page
            - $page_count: number of reachable pages
            - $items_per_page: maximal number of items per page
            - $first_item: index of first item on the current page
            - $last_item: index of last item on the current page
            - $item_count: total number of items
            - $link_first: link to first page (unless this is first page)
            - $link_last: link to last page (unless this is last page)
            - $link_previous: link to previous page (unless this is first page)
            - $link_next: link to next page (unless this is last page)

            To render a range of pages the token '~3~' can be used. The
            number sets the radius of pages around the current page.
            Example for a range with radius 3:

            '1 .. 5 6 7 [8] 9 10 11 .. 50'

            Default: '~2~'

        url
            The URL that page links will point to. Make sure it contains the string
            $page which will be replaced by the actual page number.
            Must be given unless a url_maker is specified to __init__, in which
            case this parameter is ignored.

        symbol_first
            String to be displayed as the text for the $link_first link above.

            Default: '<<' (<<)

        symbol_last
            String to be displayed as the text for the $link_last link above.

            Default: '>>' (>>)

        symbol_previous
            String to be displayed as the text for the $link_previous link above.

            Default: '<' (<)

        symbol_next
            String to be displayed as the text for the $link_next link above.

            Default: '>' (>)

        separator:
            String that is used to separate page links/numbers in the above range of pages.

            Default: ' '

        show_if_single_page:
            if True the navigator will be shown even if there is only one page.

            Default: False

        link_attr (optional)
            A dictionary of attributes that get added to A-HREF links pointing to other pages. Can
            be used to define a CSS style or class to customize the look of links.

            Example: { 'style':'border: 1px solid green' }
            Example: { 'class':'pager_link' }

        curpage_attr (optional)
            A dictionary of attributes that get added to the current page number in the pager (which
            is obviously not a link). If this dictionary is not empty then the elements will be
            wrapped in a SPAN tag with the given attributes.

            Example: { 'style':'border: 3px solid blue' }
            Example: { 'class':'pager_curpage' }

        dotdot_attr (optional)
            A dictionary of attributes that get added to the '..' string in the pager (which is
            obviously not a link). If this dictionary is not empty then the elements will be wrapped
            in a SPAN tag with the given attributes.

            Example: { 'style':'color: #808080' }
            Example: { 'class':'pager_dotdot' }

        link_tag (optional)
            A callable that accepts single argument `page` (page link information)
            and generates string with html that represents the link for specific page.
            Page objects are supplied from `link_map()` so the keys are the same.

        """
        link_attr = link_attr or {}
        curpage_attr = curpage_attr or {}
        dotdot_attr = dotdot_attr or {}
        self.curpage_attr = curpage_attr
        self.separator = separator
        self.link_attr = link_attr
        self.dotdot_attr = dotdot_attr
        self.url = url
        self.link_tag = link_tag or self.default_link_tag

        # Don't show navigator if there is no more than one page
        if self.page_count == 0 or (self.page_count == 1 and not show_if_single_page):
            return ""

        regex_res = re.search(r"~(\d+)~", format)
        if regex_res:
            radius = regex_res.group(1)
        else:
            radius = 2
        radius = int(radius)
        self.radius = radius
        link_map = self.link_map(
            format=format,
            url=url,
            show_if_single_page=show_if_single_page,
            separator=separator,
            symbol_first=symbol_first,
            symbol_last=symbol_last,
            symbol_previous=symbol_previous,
            symbol_next=symbol_next,
            link_attr=link_attr,
            curpage_attr=curpage_attr,
            dotdot_attr=dotdot_attr,
        )
        links_markup = self._range(link_map, radius)

        # Replace ~...~ in token format by range of pages
        result = re.sub(r"~(\d+)~", links_markup, format)

        link_first = (
            self.page > self.first_page and self.link_tag(link_map["first_page"]) or ""
        )
        link_last = (
            self.page < self.last_page and self.link_tag(link_map["last_page"]) or ""
        )
        link_previous = (
            self.previous_page and self.link_tag(link_map["previous_page"]) or ""
        )
        link_next = self.next_page and self.link_tag(link_map["next_page"]) or ""
        # Interpolate '$' variables
        result = Template(result).safe_substitute(
            {
                "first_page": self.first_page,
                "last_page": self.last_page,
                "page": self.page,
                "page_count": self.page_count,
                "items_per_page": self.items_per_page,
                "first_item": self.first_item,
                "last_item": self.last_item,
                "item_count": self.item_count,
                "link_first": link_first,
                "link_last": link_last,
                "link_previous": link_previous,
                "link_next": link_next,
            }
        )

        return result


Instruction:
Implement a function in Python 3 to
return a map with links to other pages.

Response:
def link_map(
        self,
        format="~2~",
        url=None,
        show_if_single_page=False,
        separator=" ",
        symbol_first="<<",
        symbol_last=">>",
        symbol_previous="<",
        symbol_next=">",
        link_attr=None,
        curpage_attr=None,
        dotdot_attr=None,
    ):
        """ Return map with links to other pages if default pager() function is not suitable solution.
        format:
            Format string that defines how the pager would be normally rendered rendered. Uses same arguments as pager()
            method, but returns a simple dictionary in form of:
            {'current_page': {'attrs': {},
                                     'href': 'http://example.org/foo/page=1',
                                     'value': 1},
                    'first_page': {'attrs': {},
                                   'href': 'http://example.org/foo/page=1',
                                   'type': 'first_page',
                                   'value': 1},
                    'last_page': {'attrs': {},
                                  'href': 'http://example.org/foo/page=8',
                                  'type': 'last_page',
                                  'value': 8},
                    'next_page': {'attrs': {}, 'href': 'HREF', 'type': 'next_page', 'value': 2},
                    'previous_page': None,
                    'range_pages': [{'attrs': {},
                                     'href': 'http://example.org/foo/page=1',
                                     'type': 'current_page',
                                     'value': 1},
                                     ....
                                    {'attrs': {}, 'href': '', 'type': 'span', 'value': '..'}]}

            The string can contain the following $-tokens that are substituted by the
            string.Template module:

            - $first_page: number of first reachable page
            - $last_page: number of last reachable page
            - $page: number of currently selected page
            - $page_count: number of reachable pages
            - $items_per_page: maximal number of items per page
            - $first_item: index of first item on the current page
            - $last_item: index of last item on the current page
            - $item_count: total number of items
            - $link_first: link to first page (unless this is first page)
            - $link_last: link to last page (unless this is last page)
            - $link_previous: link to previous page (unless this is first page)
            - $link_next: link to next page (unless this is last page)

            To render a range of pages the token '~3~' can be used. The
            number sets the radius of pages around the current page.
            Example for a range with radius 3:

            '1 .. 5 6 7 [8] 9 10 11 .. 50'

            Default: '~2~'

        url
            The URL that page links will point to. Make sure it contains the string
            $page which will be replaced by the actual page number.
            Must be given unless a url_maker is specified to __init__, in which
            case this parameter is ignored.

        symbol_first
            String to be displayed as the text for the $link_first link above.

            Default: '<<' (<<)

        symbol_last
            String to be displayed as the text for the $link_last link above.

            Default: '>>' (>>)

        symbol_previous
            String to be displayed as the text for the $link_previous link above.

            Default: '<' (<)

        symbol_next
            String to be displayed as the text for the $link_next link above.

            Default: '>' (>)

        separator:
            String that is used to separate page links/numbers in the above range of pages.

            Default: ' '

        show_if_single_page:
            if True the navigator will be shown even if there is only one page.

            Default: False

        link_attr (optional)
            A dictionary of attributes that get added to A-HREF links pointing to other pages. Can
            be used to define a CSS style or class to customize the look of links.

            Example: { 'style':'border: 1px solid green' }
            Example: { 'class':'pager_link' }

        curpage_attr (optional)
            A dictionary of attributes that get added to the current page number in the pager (which
            is obviously not a link). If this dictionary is not empty then the elements will be
            wrapped in a SPAN tag with the given attributes.

            Example: { 'style':'border: 3px solid blue' }
            Example: { 'class':'pager_curpage' }

        dotdot_attr (optional)
            A dictionary of attributes that get added to the '..' string in the pager (which is
            obviously not a link). If this dictionary is not empty then the elements will be wrapped
            in a SPAN tag with the given attributes.

            Example: { 'style':'color: #808080' }
            Example: { 'class':'pager_dotdot' }
        """
        link_attr = link_attr or {}
        curpage_attr = curpage_attr or {}
        dotdot_attr = dotdot_attr or {}
        self.curpage_attr = curpage_attr
        self.separator = separator
        self.link_attr = link_attr
        self.dotdot_attr = dotdot_attr
        self.url = url

        regex_res = re.search(r"~(\d+)~", format)
        if regex_res:
            radius = regex_res.group(1)
        else:
            radius = 2
        radius = int(radius)
        self.radius = radius

        # Compute the first and last page number within the radius
        # e.g. '1 .. 5 6 [7] 8 9 .. 12'
        # -> leftmost_page  = 5
        # -> rightmost_page = 9
        leftmost_page = (
            max(self.first_page, (self.page - radius)) if self.first_page else None
        )
        rightmost_page = (
            min(self.last_page, (self.page + radius)) if self.last_page else None
        )

        nav_items = {
            "first_page": None,
            "last_page": None,
            "previous_page": None,
            "next_page": None,
            "current_page": None,
            "radius": self.radius,
            "range_pages": [],
        }

        if leftmost_page is None or rightmost_page is None:
            return nav_items

        nav_items["first_page"] = {
            "type": "first_page",
            "value": unicode(symbol_first),
            "attrs": self.link_attr,
            "number": self.first_page,
            "href": self.url_maker(self.first_page),
        }

        # Insert dots if there are pages between the first page
        # and the currently displayed page range
        if leftmost_page - self.first_page > 1:
            # Wrap in a SPAN tag if dotdot_attr is set
            nav_items["range_pages"].append(
                {
                    "type": "span",
                    "value": "..",
                    "attrs": self.dotdot_attr,
                    "href": "",
                    "number": None,
                }
            )

        for thispage in range(leftmost_page, rightmost_page + 1):
            # Highlight the current page number and do not use a link
            if thispage == self.page:
                # Wrap in a SPAN tag if curpage_attr is set
                nav_items["range_pages"].append(
                    {
                        "type": "current_page",
                        "value": unicode(thispage),
                        "number": thispage,
                        "attrs": self.curpage_attr,
                        "href": self.url_maker(thispage),
                    }
                )
                nav_items["current_page"] = {
                    "value": thispage,
                    "attrs": self.curpage_attr,
                    "type": "current_page",
                    "href": self.url_maker(thispage),
                }
            # Otherwise create just a link to that page
            else:
                nav_items["range_pages"].append(
                    {
                        "type": "page",
                        "value": unicode(thispage),
                        "number": thispage,
                        "attrs": self.link_attr,
                        "href": self.url_maker(thispage),
                    }
                )

        # Insert dots if there are pages between the displayed
        # page numbers and the end of the page range
        if self.last_page - rightmost_page > 1:
            # Wrap in a SPAN tag if dotdot_attr is set
            nav_items["range_pages"].append(
                {
                    "type": "span",
                    "value": "..",
                    "attrs": self.dotdot_attr,
                    "href": "",
                    "number": None,
                }
            )

        # Create a link to the very last page (unless we are on the last
        # page or there would be no need to insert '..' spacers)
        nav_items["last_page"] = {
            "type": "last_page",
            "value": unicode(symbol_last),
            "attrs": self.link_attr,
            "href": self.url_maker(self.last_page),
            "number": self.last_page,
        }

        nav_items["previous_page"] = {
            "type": "previous_page",
            "value": unicode(symbol_previous),
            "attrs": self.link_attr,
            "number": self.previous_page or self.first_page,
            "href": self.url_maker(self.previous_page or self.first_page),
        }

        nav_items["next_page"] = {
            "type": "next_page",
            "value": unicode(symbol_next),
            "attrs": self.link_attr,
            "number": self.next_page or self.last_page,
            "href": self.url_maker(self.next_page or self.last_page),
        }

        return nav_items


Instruction:
Write a Python 3 script to
return a range of linked pages to substiture placeholder in pattern

Response:
def _range(self, link_map, radius):
        """
        Return range of linked pages to substiture placeholder in pattern
        """

        leftmost_page = max(self.first_page, (self.page - radius))
        rightmost_page = min(self.last_page, (self.page + radius))

        nav_items = []
        # Create a link to the first page (unless we are on the first page
        # or there would be no need to insert '..' spacers)
        if self.page != self.first_page and self.first_page < leftmost_page:
            page = link_map["first_page"].copy()
            page["value"] = unicode(page["number"])
            nav_items.append(self.link_tag(page))

        for item in link_map["range_pages"]:
            nav_items.append(self.link_tag(item))

        # Create a link to the very last page (unless we are on the last
        # page or there would be no need to insert '..' spacers)
        if self.page != self.last_page and rightmost_page < self.last_page:
            page = link_map["last_page"].copy()
            page["value"] = unicode(page["number"])
            nav_items.append(self.link_tag(page))

        return self.separator.join(nav_items)


Instruction:
Given the following Python 3 function, write the documentation
def default_link_tag(item):
        text = item["value"]
        target_url = item["href"]

        if not item["href"] or item["type"] in ("span", "current_page"):
            if item["attrs"]:
                text = make_html_tag("span", **item["attrs"]) + text + "</span>"
            return text

        return make_html_tag("a", text=text, href=target_url, **item["attrs"])

Response: Create an A - HREF tag that points to another page.


Instruction:
Make a summary of the following Python 3 code
def tag(self, corpus, tokenize=True):
        '''Tags a string `corpus`.'''
        # Assume untokenized corpus has \n between sentences and ' ' between words
        s_split = SentenceTokenizer().tokenize if tokenize else lambda t: t.split('\n')
        w_split = WordTokenizer().tokenize if tokenize else lambda s: s.split()
        def split_sents(corpus):
            for s in s_split(corpus):
                yield w_split(s)

        prev, prev2 = self.START
        tokens = []
        for words in split_sents(corpus):
            context = self.START + [self._normalize(w) for w in words] + self.END
            for i, word in enumerate(words):
                tag = self.tagdict.get(word)
                if not tag:
                    features = self._get_features(i, word, context, prev, prev2)
                    tag = self.model.predict(features)
                tokens.append((word, tag))
                prev2 = prev
                prev = tag
        return tokens

Response: Tags a string corpus.


Instruction:
Can you generate the documentation for the following Python 3 function
def train(self, sentences, save_loc=None, nr_iter=5):
        '''Train a model from sentences, and save it at ``save_loc``. ``nr_iter``
        controls the number of Perceptron training iterations.

        :param sentences: A list of (words, tags) tuples.
        :param save_loc: If not ``None``, saves a pickled model in this location.
        :param nr_iter: Number of training iterations.
        '''
        self._make_tagdict(sentences)
        self.model.classes = self.classes
        for iter_ in range(nr_iter):
            c = 0
            n = 0
            for words, tags in sentences:
                prev, prev2 = self.START
                context = self.START + [self._normalize(w) for w in words] \
                                                                    + self.END
                for i, word in enumerate(words):
                    guess = self.tagdict.get(word)
                    if not guess:
                        feats = self._get_features(i, word, context, prev, prev2)
                        guess = self.model.predict(feats)
                        self.model.update(tags[i], guess, feats)
                    prev2 = prev
                    prev = guess
                    c += guess == tags[i]
                    n += 1
            random.shuffle(sentences)
            logging.info("Iter {0}: {1}/{2}={3}".format(iter_, c, n, _pc(c, n)))
        self.model.average_weights()
        # Pickle as a binary file
        if save_loc is not None:
            pickle.dump((self.model.weights, self.tagdict, self.classes),
                         open(save_loc, 'wb'), -1)
        return None

Response: Train a model from sentences and save it at save_loc.


Instruction:
Make a summary of the following Python 3 code
def load(self, loc):
        '''Load a pickled model.'''
        try:
            w_td_c = pickle.load(open(loc, 'rb'))
        except IOError:
            msg = ("Missing trontagger.pickle file.")
            raise MissingCorpusError(msg)
        self.model.weights, self.tagdict, self.classes = w_td_c
        self.model.classes = self.classes
        return None

Response: Load a pickled model.


Instruction:
Make a summary of the following Python 3 code
def _normalize(self, word):
        '''Normalization used in pre-processing.

        - All words are lower cased
        - Digits in the range 1800-2100 are represented as !YEAR;
        - Other digits are represented as !DIGITS

        :rtype: str
        '''
        if '-' in word and word[0] != '-':
            return '!HYPHEN'
        elif word.isdigit() and len(word) == 4:
            return '!YEAR'
        elif word[0].isdigit():
            return '!DIGITS'
        else:
            return word.lower()

Response: Normalizes used in pre - processing.


Instruction:
Can you tell what is the following Python 3 function doing
def _get_features(self, i, word, context, prev, prev2):
        '''Map tokens into a feature representation, implemented as a
        {hashable: float} dict. If the features change, a new model must be
        trained.
        '''
        def add(name, *args):
            features[' '.join((name,) + tuple(args))] += 1

        i += len(self.START)
        features = defaultdict(int)
        # It's useful to have a constant feature, which acts sort of like a prior
        add('bias')
        add('i suffix', word[-3:])
        add('i pref1', word[0])
        add('i-1 tag', prev)
        add('i-2 tag', prev2)
        add('i tag+i-2 tag', prev, prev2)
        add('i word', context[i])
        add('i-1 tag+i word', prev, context[i])
        add('i-1 word', context[i-1])
        add('i-1 suffix', context[i-1][-3:])
        add('i-2 word', context[i-2])
        add('i+1 word', context[i+1])
        add('i+1 suffix', context[i+1][-3:])
        add('i+2 word', context[i+2])
        return features

Response:
Map tokens into a feature representation implemented as a
        dict.


Instruction:
Implement a function in Python 3 to
make a tag dictionary for single - tag words.

Response:
def _make_tagdict(self, sentences):
        '''Make a tag dictionary for single-tag words.'''
        counts = defaultdict(lambda: defaultdict(int))
        for words, tags in sentences:
            for word, tag in zip(words, tags):
                counts[word][tag] += 1
                self.classes.add(tag)
        freq_thresh = 20
        ambiguity_thresh = 0.97
        for word, tag_freqs in counts.items():
            tag, mode = max(tag_freqs.items(), key=lambda item: item[1])
            n = sum(tag_freqs.values())
            # Don't add rare words to the tag dictionary
            # Only add quite unambiguous words
            if n >= freq_thresh and (float(mode) / n) >= ambiguity_thresh:
                self.tagdict[word] = tag


Instruction:
Write a Python 3 script for
training an averaged perceptron model on the given examples.

Response:
def train(nr_iter, examples):
    '''Return an averaged perceptron model trained on ``examples`` for
    ``nr_iter`` iterations.
    '''
    model = AveragedPerceptron()
    for i in range(nr_iter):
        random.shuffle(examples)
        for features, class_ in examples:
            scores = model.predict(features)
            guess, score = max(scores.items(), key=lambda i: i[1])
            if guess != class_:
                model.update(class_, guess, features)
    model.average_weights()
    return model


Instruction:
Can you generate the documentation for the following Python 3 function
def encode(self, V, P, X, CC, seqNum, M, PT, SSRC, payload):
        timestamp = int(time())
        header = bytearray(HEADER_SIZE)
        # Fill the header bytearray with RTP header fields
        # ...
        header[0] = header[0] | V << 6;
        header[0] = header[0] | P << 5;
        header[0] = header[0] | X << 4;
        header[0] = header[0] | CC;
        header[1] = header[1] | M << 7;
        header[1] = header[1] | PT;
        header[2] = (seqNum >> 8) & 0xFF;
        header[3] = seqNum & 0xFF;
        header[4] = (timestamp >> 24) & 0xFF;
        header[5] = (timestamp >> 16) & 0xFF;
        header[6] = (timestamp >> 8) & 0xFF;
        header[7] = timestamp & 0xFF;
        header[8] = (SSRC >> 24) & 0xFF;
        header[9] = (SSRC >> 16) & 0xFF;
        header[10] = (SSRC >> 8) & 0xFF;
        header[11] = SSRC & 0xFF

        self.header = header

        # Get the payload
        # ...
        self.payload = payload

Response: Encode the RTP packet with header fields and payload.


Instruction:
Create a Python 3 function for
decoding the RTP packet.

Response:
def decode(self, byteStream):
        """Decode the RTP packet."""
        self.header = bytearray(byteStream[:HEADER_SIZE])
        self.payload = byteStream[HEADER_SIZE:]


Instruction:
Given the following Python 3 function, write the documentation
def timestamp(self):
        timestamp = self.header[4] << 24 | self.header[5] << 16 | self.header[6] << 8 | self.header[7]
        return int(timestamp)

Response: Return the timestamp of the record.


Instruction:
Make a summary of the following Python 3 code
def preview_stream(stream):
    # together with waitkeys later, helps to close the video window effectively
    _cv2.startWindowThread()

    for frame in stream.frame_generator():
        if frame is not None:
            _cv2.imshow('Video', frame)
            _cv2.moveWindow('Video',5,5)
        else:
            break
        key = _cv2.waitKey(1) & 0xFF
        if key == ord("q"):
            break
    _cv2.waitKey(1)
    _cv2.destroyAllWindows()
    _cv2.waitKey(1)

Response: Display a stream in an OpenCV window until q key is pressed.


Instruction:
Can you create a Python 3 function that
gets a list of available resources from the RTSP - DESCRIBE response.

Response:
def get_resources(connection):
    """ Do an RTSP-DESCRIBE request, then parse out available resources from the response """
    resp = connection.describe(verbose=False).split('\r\n')
    resources = [x.replace('a=control:','') for x in resp if (x.find('control:') != -1 and x[-1] != '*' )]
    return resources


Instruction:
How would you explain what the following Python 3 function does
def setup(self,resource_path = "track1"):
        self.rtsp_seq += 1

        uri = '/'.join([s.strip('/') for s in (self.server,self.stream_path,resource_path)])

        ## example: SETUP rtsp://example.com/foo/bar/baz.rm RTSP/2.0
        request = f"SETUP rtsp://{uri} {RTSP_VER}\r\n"
        request+= f"CSeq: {self.rtsp_seq}\r\n"
        request+= f"Transport: RTP/AVP;unicast;client_port=5000-5001\r\n"#,RTP/SAVPF,RTP/AVP;unicast;client_port=5000-5001,RTP/AVP/UDP;unicast;client_port=5000-5001\r\n"
        #request+= f"Accept-Ranges: npt, smpte, clock\r\n"
        request+= f"User-Agent: python\r\n\r\n"

        reply = self.sendMessage(request.encode('UTF-8'))
        return reply

Response: Send SETUP request to set up the current state of the current node.


Instruction:
Create a Python 3 function to
fetch a single frame using FFMPEG. Convert to PIL Image. Slow.

Response:
def fetch_image(self,rtsp_server_uri = _source,timeout_secs = 15):
        """ Fetch a single frame using FFMPEG. Convert to PIL Image. Slow. """

        self._check_ffmpeg()
        cmd = "ffmpeg -rtsp_transport tcp -i {} -loglevel quiet -frames 1 -f image2pipe -".format(rtsp_server_uri)

        #stdout = _sp.check_output(ffmpeg_cmd,timeout = timeout_secs)
        with _sp.Popen(cmd, shell=True,  stdout=_sp.PIPE) as process:
            try:
                stdout,stderr = process.communicate(timeout=timeout_secs)
            except _sp.TimeoutExpired as e:
                process.kill()
                raise TimeoutError("Connection to {} timed out".format(rtsp_server_uri),e)

        return _Image.open(_io.BytesIO(stdout))


Instruction:
Create a Python 3 function to
execute a command in the current thread.

Response:
def exec_cmd(rtsp,cmd):
    '''根据命令执行操作'''
    global CUR_RANGE,CUR_SCALE
    if cmd in ('exit','teardown'):
        rtsp.do_teardown()
    elif cmd == 'pause':
        CUR_SCALE = 1; CUR_RANGE = 'npt=now-'
        rtsp.do_pause()
    elif cmd == 'help':
        PRINT(play_ctrl_help())
    elif cmd == 'forward':
        if CUR_SCALE < 0: CUR_SCALE = 1
        CUR_SCALE *= 2; CUR_RANGE = 'npt=now-'
    elif cmd == 'backward':
        if CUR_SCALE > 0: CUR_SCALE = -1
        CUR_SCALE *= 2; CUR_RANGE = 'npt=now-'
    elif cmd == 'begin':
        CUR_SCALE = 1; CUR_RANGE = 'npt=beginning-'
    elif cmd == 'live':
        CUR_SCALE = 1; CUR_RANGE = 'npt=end-'
    elif cmd.startswith('play'):
        m = re.search(r'range[:\s]+(?P<range>[^\s]+)',cmd)
        if m: CUR_RANGE = m.group('range')
        m = re.search(r'scale[:\s]+(?P<scale>[\d\.]+)',cmd)
        if m: CUR_SCALE = int(m.group('scale'))

    if cmd not in ('pause','exit','teardown','help'):
        rtsp.do_play(CUR_RANGE,CUR_SCALE)


Instruction:
Implement a Python 3 function for
parsing url and return ip port target

Response:
def _parse_url(self,url):
        '''解析url,返回(ip,port,target)三元组'''
        (ip,port,target) = ('',DEFAULT_SERVER_PORT,'')
        m = re.match(r'[rtspRTSP:/]+(?P<ip>(\d{1,3}\.){3}\d{1,3})(:(?P<port>\d+))?(?P<target>.*)',url)
        if m is not None:
            ip      = m.group('ip')
            port    = int(m.group('port'))
            target  = m.group('target')
        #PRINT('ip: %s, port: %d, target: %s'%(ip,port,target), GREEN)
        return ip,port,target


Instruction:
How would you code a function in Python 3 to
connect to the server and return the ID.

Response:
def _connect_server(self):
        '''连接服务器,建立socket'''
        try:
            self._sock = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
            self._sock.connect((self._server_ip,self._server_port))
            #PRINT('Connect [%s:%d] success!'%(self._server_ip,self._server_port), GREEN)
        except socket.error, e:
            sys.stderr.write('ERROR: %s[%s:%d]'%(e,self._server_ip,self._server_port))
            traceback.print_exc()
            sys.exit(1)


Instruction:
Write a Python 3 function for
updating the DEST_IP global variable.

Response:
def _update_dest_ip(self):
        '''如果未指定DEST_IP,默认与RTSP使用相同IP'''
        global DEST_IP
        if not DEST_IP:
            DEST_IP = self._sock.getsockname()[0]
            PRINT('DEST_IP: %s\n'%DEST_IP, CYAN)


Instruction:
Can you write a function in Python 3 where it
receives a message from the socket.

Response:
def recv_msg(self):
        '''收取一个完整响应消息或ANNOUNCE通知消息'''
        try:
            while True:
                if HEADER_END_STR in self._recv_buf: break
                more = self._sock.recv(2048)
                if not more: break
                self._recv_buf += more
        except socket.error, e:
            PRINT('Receive data error: %s'%e,RED)
            sys.exit(-1)

        msg = ''
        if self._recv_buf:
            (msg,self._recv_buf) = self._recv_buf.split(HEADER_END_STR,1)
            content_length = self._get_content_length(msg)
            msg += HEADER_END_STR + self._recv_buf[:content_length]
            self._recv_buf = self._recv_buf[content_length:]
        return msg


Instruction:
Implement a function in Python 3 to
get the content length from the message.

Response:
def _get_content_length(self,msg):
        '''从消息中解析Content-length'''
        m = re.search(r'[Cc]ontent-length:\s?(?P<len>\d+)',msg,re.S)
        return (m and int(m.group('len'))) or 0


Instruction:
How would you code a function in Python 3 to
process a response from the server.

Response:
def _process_response(self,msg):
        '''处理响应消息'''
        status,headers,body = self._parse_response(msg)
        rsp_cseq = int(headers['cseq'])
        if self._cseq_map[rsp_cseq] != 'GET_PARAMETER':
            PRINT(self._get_time_str() + '\n' + msg)
        if status == 302:
            self.location = headers['location']
        if status != 200:
            self.do_teardown()
        if self._cseq_map[rsp_cseq] == 'DESCRIBE':
            track_id_str = self._parse_track_id(body)
            self.do_setup(track_id_str)
        elif self._cseq_map[rsp_cseq] == 'SETUP':
            self._session_id = headers['session']
            self.do_play(CUR_RANGE,CUR_SCALE)
            self.send_heart_beat_msg()
        elif self._cseq_map[rsp_cseq] == 'PLAY':
            self.playing = True


Instruction:
Create a Python 3 function to
process an announcing message.

Response:
def _process_announce(self,msg):
        '''处理ANNOUNCE通知消息'''
        global CUR_RANGE,CUR_SCALE
        PRINT(msg)
        headers = self._parse_header_params(msg.splitlines()[1:])
        x_notice_val = int(headers['x-notice'])
        if x_notice_val in (X_NOTICE_EOS,X_NOTICE_BOS):
            CUR_SCALE = 1
            self.do_play(CUR_RANGE,CUR_SCALE)
        elif x_notice_val == X_NOTICE_CLOSE:
            self.do_teardown()


Instruction:
How would you explain what the following Python 3 function does
def _parse_response(self,msg):
        '''解析响应消息'''
        header,body = msg.split(HEADER_END_STR)[:2]
        header_lines = header.splitlines()
        version,status = header_lines[0].split(None,2)[:2]
        headers = self._parse_header_params(header_lines[1:])
        return int(status),headers,body

Response: Parse the response from the server.


Instruction:
How would you implement a function in Python 3 that
parses the header parameter lines and return a dict of the header keys and values.

Response:
def _parse_header_params(self,header_param_lines):
        '''解析头部参数'''
        headers = {}
        for line in header_param_lines:
            if line.strip(): # 跳过空行
                key,val = line.split(':', 1)
                headers[key.lower()] = val.strip()
        return headers


Instruction:
Can you create a Python 3 function that
parses the track ID from the SDP

Response:
def _parse_track_id(self,sdp):
        '''从sdp中解析trackID=2形式的字符串'''
        m = re.search(r'a=control:(?P<trackid>[\w=\d]+)',sdp,re.S)
        return (m and m.group('trackid')) or ''


Instruction:
Explain what the following Python 3 code does
def _sendmsg(self,method,url,headers):
        '''发送消息'''
        msg = '%s %s %s'%(method,url,RTSP_VERSION)
        headers['User-Agent'] = DEFAULT_USERAGENT
        cseq = self._next_seq()
        self._cseq_map[cseq] = method
        headers['CSeq'] = str(cseq)
        if self._session_id: headers['Session'] = self._session_id
        for (k,v) in headers.items():
            msg += LINE_SPLIT_STR + '%s: %s'%(k,str(v))
        msg += HEADER_END_STR # End headers
        if method != 'GET_PARAMETER' or 'x-RetransSeq' in headers:
            PRINT(self._get_time_str() + LINE_SPLIT_STR + msg)
        try:
            self._sock.send(msg)
        except socket.error, e:
            PRINT('Send msg error: %s'%e, RED)

Response: Send a message to the server.


Instruction:
Write a Python 3 script for
getting the transport type

Response:
def _get_transport_type(self):
        '''获取SETUP时需要的Transport字符串参数'''
        transport_str = ''
        ip_type = 'unicast' #if IPAddress(DEST_IP).is_unicast() else 'multicast'
        for t in TRANSPORT_TYPE_LIST:
            if t not in TRANSPORT_TYPE_MAP:
                PRINT('Error param: %s'%t,RED)
                sys.exit(1)
            if t.endswith('tcp'):
                transport_str += TRANSPORT_TYPE_MAP[t]%ip_type
            else:
                transport_str += TRANSPORT_TYPE_MAP[t]%(ip_type,DEST_IP,CLIENT_PORT_RANGE)
        return transport_str


Instruction:
Create a Python 3 function to
send a heartbeat message to the server

Response:
def send_heart_beat_msg(self):
        '''定时发送GET_PARAMETER消息保活'''
        if self.running:
            self.do_get_parameter()
            threading.Timer(HEARTBEAT_INTERVAL, self.send_heart_beat_msg).start()


Instruction:
Can you generate a brief explanation for the following Python 3 code
def updateMovie(self, imageFile):
    try:
      photo = ImageTk.PhotoImage(Image.open(imageFile)) #stuck here !!!!!!
    except:
      print("photo error")
      print('-'*60)
      traceback.print_exc(file=sys.stdout)
      print('-'*60)

    self.label.configure(image = photo, height=288)
    self.label.image = photo

Response: Update the image file as video frame in the GUI.


Instruction:
Implement a function in Python 3 to
connect to the Server.

Response:
def connectToServer(self):
    """Connect to the Server. Start a new RTSP/TCP session."""
    self.rtspSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    try:
      self.rtspSocket.connect((self.serverAddr, self.serverPort))
    except:
      tkinter.messagebox.showwarning('Connection Failed', 'Connection to \'%s\' failed.' %self.serverAddr)


Instruction:
Here you have a function in Python 3, explain what it does
def sendRtspRequest(self, requestCode):
    #-------------
    # TO COMPLETE
    #-------------

    # Setup request
    if requestCode == self.SETUP and self.state == self.INIT:
      threading.Thread(target=self.recvRtspReply).start()
      # Update RTSP sequence number.
      # ...
      self.rtspSeq = 1

      # Write the RTSP request to be sent.
      # request = ...
      request = "SETUP " + str(self.fileName) + "\n" + str(self.rtspSeq) + "\n" + " RTSP/1.0 RTP/UDP " + str(self.rtpPort)

      self.rtspSocket.send(request)
      # Keep track of the sent request.
      # self.requestSent = ...
      self.requestSent = self.SETUP

    # Play request
    elif requestCode == self.PLAY and self.state == self.READY:
      # Update RTSP sequence number.
      # ...
      self.rtspSeq = self.rtspSeq + 1
      # Write the RTSP request to be sent.
      # request = ...
      request = "PLAY " + "\n" + str(self.rtspSeq)

      self.rtspSocket.send(request)
      print('-'*60 + "\nPLAY request sent to Server...\n" + '-'*60)
      # Keep track of the sent request.
      # self.requestSent = ...
      self.requestSent = self.PLAY

    # Pause request
    elif requestCode == self.PAUSE and self.state == self.PLAYING:
      # Update RTSP sequence number.
      # ...
      self.rtspSeq = self.rtspSeq + 1
      # Write the RTSP request to be sent.
      # request = ...
      request = "PAUSE " + "\n" + str(self.rtspSeq)
      self.rtspSocket.send(request)
      print('-'*60 + "\nPAUSE request sent to Server...\n" + '-'*60)
      # Keep track of the sent request.
      # self.requestSent = ...
      self.requestSent = self.PAUSE

    # Resume request

    # Teardown request
    elif requestCode == self.TEARDOWN and not self.state == self.INIT:
      # Update RTSP sequence number.
      # ...
      self.rtspSeq = self.rtspSeq + 1
      # Write the RTSP request to be sent.
      # request = ...
      request = "TEARDOWN " + "\n" + str(self.rtspSeq)
      self.rtspSocket.send(request)
      print('-'*60 + "\nTEARDOWN request sent to Server...\n" + '-'*60)
      # Keep track of the sent request.
      # self.requestSent = ...
      self.requestSent = self.TEARDOWN

    else:
      return

Response: Send RTSP request to the server.


Instruction:
Implement a function in Python 3 to
receive RTSP reply from the server.

Response:
def recvRtspReply(self):
    """Receive RTSP reply from the server."""
    while True:
      reply = self.rtspSocket.recv(1024)

      if reply:
        self.parseRtspReply(reply)

      # Close the RTSP socket upon requesting Teardown
      if self.requestSent == self.TEARDOWN:
        self.rtspSocket.shutdown(socket.SHUT_RDWR)
        self.rtspSocket.close()
        break


Instruction:
How would you explain what the following Python 3 function does
def parseRtspReply(self, data):
    print("Parsing Received Rtsp data...")

    """Parse the RTSP reply from the server."""
    lines = data.split('\n')
    seqNum = int(lines[1].split(' ')[1])

    # Process only if the server reply's sequence number is the same as the request's
    if seqNum == self.rtspSeq:
      session = int(lines[2].split(' ')[1])
      # New RTSP session ID
      if self.sessionId == 0:
        self.sessionId = session

      # Process only if the session ID is the same
      if self.sessionId == session:
        if int(lines[0].split(' ')[1]) == 200:
          if self.requestSent == self.SETUP:
            #-------------
            # TO COMPLETE
            #-------------
            # Update RTSP state.
            print("Updating RTSP state...")
            # self.state = ...
            self.state = self.READY
            # Open RTP port.
            #self.openRtpPort()
            print("Setting Up RtpPort for Video Stream")
            self.openRtpPort()

          elif self.requestSent == self.PLAY:
             self.state = self.PLAYING
             print('-'*60 + "\nClient is PLAYING...\n" + '-'*60)
          elif self.requestSent == self.PAUSE:
             self.state = self.READY

            # The play thread exits. A new thread is created on resume.
             self.playEvent.set()

          elif self.requestSent == self.TEARDOWN:
            # self.state = ...

            # Flag the teardownAcked to close the socket.
            self.teardownAcked = 1

Response: Parse the RTSP reply from the server.


Instruction:
Implement a Python 3 function for
opening a datagram socket binded to a specified port.

Response:
def openRtpPort(self):
    """Open RTP socket binded to a specified port."""
    #-------------
    # TO COMPLETE
    #-------------
    # Create a new datagram socket to receive RTP packets from the server
    # self.rtpSocket = ...

    # Set the timeout value of the socket to 0.5sec
    # ...
    self.rtpSocket.settimeout(0.5)
#   try:
      # Bind the socket to the address using the RTP port given by the client user
      # ...
#   except:
#     tkMessageBox.showwarning('Unable to Bind', 'Unable to bind PORT=%d' %self.rtpPort)

    try:
      #self.rtpSocket.connect(self.serverAddr,self.rtpPort)
      self.rtpSocket.bind((self.serverAddr,self.rtpPort))   # WATCH OUT THE ADDRESS FORMAT!!!!!  rtpPort# should be bigger than 1024
      #self.rtpSocket.listen(5)
      print("Bind RtpPort Success")

    except:
      tkinter.messagebox.showwarning('Connection Failed', 'Connection to rtpServer failed...')


Instruction:
Create a Python 3 function to
read the image from the camera.

Response:
def read(self):
        """https://picamera.readthedocs.io/en/release-1.13/recipes1.html#capturing-to-a-pil-image"""
        stream = BytesIO()
        self.cam.capture(stream, format='png')
        # "Rewind" the stream to the beginning so we can read its content
        stream.seek(0)
        return Image.open(stream)