Instruction:
Implement a Python 3 function for
writing a single java message into a log file.

Response:
def write_java_message(key,val,text_file):
    """
    Loop through all java messages that are not associated with a unit test and
    write them into a log file.

    Parameters
    ----------
    key :  str
        9.general_bad_java_messages
    val : list of list of str
        contains the bad java messages and the message types.

    :return: none
    """

    text_file.write(key)
    text_file.write('\n')

    if (len(val[0]) > 0) and (len(val) >= 3):
        for index in range(len(val[0])):
            text_file.write("Java Message Type: ")
            text_file.write(val[1][index])
            text_file.write('\n')

            text_file.write("Java Message: ")

            for jmess in val[2][index]:
                text_file.write(jmess)
                text_file.write('\n')

        text_file.write('\n \n')


Instruction:
Can you create a Python 3 function that
loads in pickle file that contains dict with bad java messages to ignore per unit test or for all cases.

Response:
def load_java_messages_to_ignore():
    """
    Load in pickle file that contains dict structure with bad java messages to ignore per unit test
    or for all cases.  The ignored bad java info is stored in g_ok_java_messages dict.

    :return:
    """
    global g_ok_java_messages
    global g_java_message_pickle_filename

    if os.path.isfile(g_java_message_pickle_filename):
        with open(g_java_message_pickle_filename,'rb') as tfile:
            g_ok_java_messages = pickle.load(tfile)
    else:
        g_ok_java_messages["general"] = []


Instruction:
Given the following Python 3 function, write the documentation
def main(argv):
    global g_script_name
    global g_test_root_dir
    global g_temp_filename
    global g_output_filename_failed_tests
    global g_output_filename_passed_tests
    global g_output_pickle_filename
    global g_failure_occurred
    global g_failed_test_info_dict
    global g_java_message_pickle_filename
    global g_summary_text_filename

    if len(argv) < 3:
        print "Must resource url like http://mr-0xa1:8080/view/wendy_jenkins/job/h2o_regression_pyunit_medium_large/lastBuild/consoleFull, filename of summary text, filename (optional ending in .pickle) to retrieve Java error messages to exclude.\n"
        sys.exit(1)
    else:   # we may be in business
        g_script_name = os.path.basename(argv[0])   # get name of script being run.
        resource_url = argv[1]

        g_temp_filename = os.path.join(g_test_root_dir,'tempText')
        g_summary_text_filename = os.path.join(g_test_root_dir,argv[2])

        if len(argv) == 4:
            g_java_message_pickle_filename  = argv[3]

        get_console_out(resource_url)       # save remote console output in local directory
        extract_job_build_url(resource_url) # extract the job name of build id for identification purposes

        log_filename = g_failed_test_info_dict["1.jobName"]
        log_pickle_filename = g_failed_test_info_dict["1.jobName"]

        # pickle file that store bad Java messages that we can ignore.
        g_java_message_pickle_filename = os.path.join(g_test_root_dir,g_java_message_pickle_filename)
        g_output_filename_failed_tests = os.path.join(g_test_root_dir,log_filename)
        g_output_filename_passed_tests = os.path.join(g_test_root_dir,log_filename)
        g_output_pickle_filename = os.path.join(g_test_root_dir,log_pickle_filename)

        load_java_messages_to_ignore()          # load in bad java messages to ignore and store in g_ok_java_messages
        extract_test_results()      # grab the console text and stored the failed tests.
        extract_java_messages()     # grab dangerous java messages that we found for the various unit tests
        if ((len(g_failed_jobs) > 0) or (g_failed_test_info_dict["7.build_failure"]=='Yes')):
            g_failure_occurred = True

        if g_failure_occurred:
            save_dict() # save the dict structure in a pickle file and a text file when failure is detected
            update_summary_file()   # join together all log files into one giant summary text.

            # output this info to console to form the list of failed jenkins jobs.
            print g_failed_test_info_dict["1.jobName"]+' build '+g_failed_test_info_dict["2.build_id"]+','
        else:
            print ""

Response: Main function for the non - fatal test case.


Instruction:
Here you have a function in Python 3, explain what it does
def normalize_enum_constant(s):
    if s.islower(): return s
    if s.isupper(): return s.lower()
    return "".join(ch if ch.islower() else "_" + ch.lower() for ch in s).strip("_")

Response: Return enum constant s converted to a canonical snake - case.


Instruction:
How would you code a function in Python 3 to
find synonyms using a word2vec model.

Response:
def find_synonyms(self, word, count=20):
        """
        Find synonyms using a word2vec model.

        :param str word: A single word to find synonyms for.
        :param int count: The first "count" synonyms will be returned.

        :returns: the approximate reconstruction of the training data.
        """
        j = h2o.api("GET /3/Word2VecSynonyms", data={'model': self.model_id, 'word': word, 'count': count})
        return OrderedDict(sorted(zip(j['synonyms'], j['scores']), key=lambda t: t[1], reverse=True))


Instruction:
How would you explain what the following Python 3 function does
def transform(self, words, aggregate_method):
        j = h2o.api("GET /3/Word2VecTransform", data={'model': self.model_id, 'words_frame': words.frame_id, 'aggregate_method': aggregate_method})
        return h2o.get_frame(j["vectors_frame"]["name"])

Response: Transform words or sequences of words to vectors using a word2vec model.


Instruction:
Write a Python 3 function for
polling the job for the next time.

Response:
def poll(self, verbose_model_scoring_history = False):
        """
        Wait until the job finishes.

        This method will continuously query the server about the status of the job, until the job reaches a
        completion. During this time we will display (in stdout) a progress bar with % completion status.
        """
        try:
            hidden = not H2OJob.__PROGRESS_BAR__
            pb = ProgressBar(title=self._job_type + " progress", hidden=hidden)
            if verbose_model_scoring_history:
                pb.execute(self._refresh_job_status, print_verbose_info=lambda x: self._print_verbose_info() if int(x * 10) % 5 == 0  else " ")
            else:
                pb.execute(self._refresh_job_status)
        except StopIteration as e:
            if str(e) == "cancelled":
                h2o.api("POST /3/Jobs/%s/cancel" % self.job_key)
                self.status = "CANCELLED"
            # Potentially we may want to re-raise the exception here

        assert self.status in {"DONE", "CANCELLED", "FAILED"} or self._poll_count <= 0, \
            "Polling finished while the job has status %s" % self.status
        if self.warnings:
            for w in self.warnings:
                warnings.warn(w)

        # check if failed... and politely print relevant message
        if self.status == "CANCELLED":
            raise H2OJobCancelled("Job<%s> was cancelled by the user." % self.job_key)
        if self.status == "FAILED":
            if (isinstance(self.job, dict)) and ("stacktrace" in list(self.job)):
                raise EnvironmentError("Job with key {} failed with an exception: {}\nstacktrace: "
                                       "\n{}".format(self.job_key, self.exception, self.job["stacktrace"]))
            else:
                raise EnvironmentError("Job with key %s failed with an exception: %s" % (self.job_key, self.exception))

        return self


Instruction:
Create a Python 3 function for
converting the munging operations performed on H2OFrame into a POJO.

Response:
def to_pojo(self, pojo_name="", path="", get_jar=True):
        """
        Convert the munging operations performed on H2OFrame into a POJO.

        :param pojo_name:  (str) Name of POJO
        :param path:  (str) path of POJO.
        :param get_jar: (bool) Whether to also download the h2o-genmodel.jar file needed to compile the POJO
        :return: None
        """
        assert_is_type(pojo_name, str)
        assert_is_type(path, str)
        assert_is_type(get_jar, bool)
        if pojo_name == "":
            pojo_name = "AssemblyPOJO_" + str(uuid.uuid4())
        java = h2o.api("GET /99/Assembly.java/%s/%s" % (self.id, pojo_name))
        file_path = path + "/" + pojo_name + ".java"
        if path == "":
            print(java)
        else:
            with open(file_path, 'w', encoding="utf-8") as f:
                f.write(java)  # this had better be utf-8 ?
        if get_jar and path != "":
            h2o.api("GET /3/h2o-genmodel.jar", save_to=os.path.join(path, "h2o-genmodel.jar"))


Instruction:
Here you have a function in Python 3, explain what it does
def fit(self, fr):
        assert_is_type(fr, H2OFrame)
        steps = "[%s]" % ",".join(quoted(step[1].to_rest(step[0]).replace('"', "'")) for step in self.steps)
        j = h2o.api("POST /99/Assembly", data={"steps": steps, "frame": fr.frame_id})
        self.id = j["assembly"]["name"]
        return H2OFrame.get_frame(j["result"]["name"])

Response: Fits the munging operations on a frame.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def percentileOnSortedList(N, percent, key=lambda x:x, interpolate='mean'):
    # 5 ways of resolving fractional
    # floor, ceil, funky, linear, mean
    interpolateChoices = ['floor', 'ceil', 'funky', 'linear', 'mean']
    if interpolate not in interpolateChoices:
        print "Bad choice for interpolate:", interpolate
        print "Supported choices:", interpolateChoices
    """
    Find the percentile of a list of values.

    @parameter N - is a list of values. Note N MUST BE already sorted.
    @parameter percent - a float value from 0.0 to 1.0.
    @parameter key - optional key function to compute value from each element of N.

    @return - the percentile of the values
    """
    if N is None:
        return None
    k = (len(N)-1) * percent
    f = int(math.floor(k))
    c = int(math.ceil(k))

    if f == c:
        d = key(N[f])
        msg = "aligned:"

    elif interpolate=='floor':
        d = key(N[f])
        msg = "fractional with floor:"

    elif interpolate=='ceil':
        d = key(N[c])
        msg = "fractional with ceil:"

    elif interpolate=='funky':
        d0 = key(N[f]) * (c-k)
        d1 = key(N[c]) * (k-f)
        d = d0+d1
        msg = "fractional with Tung(floor and ceil) :"

    elif interpolate=='linear':
        assert (c-f)==1
        assert (k>=f) and (k<=c)
        pctDiff = k-f
        dDiff = pctDiff * (key(N[c]) - key(N[f]))
        d = key(N[f] + dDiff)
        msg = "fractional %s with linear(floor and ceil):" % pctDiff

    elif interpolate=='mean':
        d = (key(N[c]) + key(N[f])) / 2.0
        msg = "fractional with mean(floor and ceil):"

    # print 3 around the floored k, for eyeballing when we're close
    flooredK = int(f)
    # print the 3 around the median
    if flooredK > 0:
        print "prior->", key(N[flooredK-1]), " "
    else:
        print "prior->", "<bof>"
    print "floor->", key(N[flooredK]), " ", msg, 'result:', d, "f:", f, "len(N):", len(N)
    if flooredK+1 < len(N):
        print " ceil->", key(N[flooredK+1]), "c:", c
    else:
        print " ceil-> <eof>", "c:", c

    return d

Response: This function returns the percentile of a list of values.


Instruction:
Can you tell what is the following Python 3 function doing
def params(self):
        params = {}
        for p in self.parms:
            params[p] = {"default": self.parms[p]["default_value"],
                         "actual": self.parms[p]["actual_value"]}
        return params

Response: Get the parameters and the actual values only.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def default_params(self):
        params = {}
        for p in self.parms:
            params[p] = self.parms[p]["default_value"]
        return params

Response: Dictionary of the default parameters of the model.


Instruction:
Can you generate the documentation for the following Python 3 function
def actual_params(self):
        params_to_select = {"model_id": "name",
                            "response_column": "column_name",
                            "training_frame": "name",
                            "validation_frame": "name"}
        params = {}
        for p in self.parms:
            if p in params_to_select.keys():
                params[p] = self.parms[p]["actual_value"].get(params_to_select[p], None)
            else:
                params[p] = self.parms[p]["actual_value"]
        return params

Response: Dictionary of actual parameters of the model.


Instruction:
Given the following Python 3 function, write the documentation
def predict_leaf_node_assignment(self, test_data, type="Path"):
        if not isinstance(test_data, h2o.H2OFrame): raise ValueError("test_data must be an instance of H2OFrame")
        assert_is_type(type, None, Enum("Path", "Node_ID"))
        j = h2o.api("POST /3/Predictions/models/%s/frames/%s" % (self.model_id, test_data.frame_id),
                    data={"leaf_node_assignment": True, "leaf_node_assignment_type": type})
        return h2o.get_frame(j["predictions_frame"]["name"])

Response: Predict on a dataset and return the leaf node assignment.


Instruction:
Can you generate the documentation for the following Python 3 function
def staged_predict_proba(self, test_data):
        if not isinstance(test_data, h2o.H2OFrame): raise ValueError("test_data must be an instance of H2OFrame")
        j = h2o.api("POST /3/Predictions/models/%s/frames/%s" % (self.model_id, test_data.frame_id),
                    data={"predict_staged_proba": True})
        return h2o.get_frame(j["predictions_frame"]["name"])

Response: Predict class probabilities at each stage of an H2O Model.


Instruction:
Make a summary of the following Python 3 code
def predict(self, test_data, custom_metric = None, custom_metric_func = None):
        # Upload evaluation function into DKV
        if custom_metric:
            assert_satisfies(custom_metric_func, custom_metric_func is None,
                             "The argument 'eval_func_ref' cannot be specified when eval_func is specified, ")
            eval_func_ref = h2o.upload_custom_metric(custom_metric)
        if not isinstance(test_data, h2o.H2OFrame): raise ValueError("test_data must be an instance of H2OFrame")
        j = H2OJob(h2o.api("POST /4/Predictions/models/%s/frames/%s" % (self.model_id, test_data.frame_id), data = {'custom_metric_func': custom_metric_func}),
                   self._model_json["algo"] + " prediction")
        j.poll()
        return h2o.get_frame(j.dest_key)

Response: Predict on a dataset.


Instruction:
Implement a function in Python 3 to
return a list of models that are cross - validated by the given key.

Response:
def get_xval_models(self, key=None):
        """
        Return a Model object.

        :param key: If None, return all cross-validated models; otherwise return the model that key points to.

        :returns: A model or list of models.
        """
        return h2o.get_model(key) if key is not None else [h2o.get_model(k) for k in self._xval_keys]


Instruction:
Make a summary of the following Python 3 code
def deepfeatures(self, test_data, layer):
        if test_data is None: raise ValueError("Must specify test data")
        if str(layer).isdigit():
            j = H2OJob(h2o.api("POST /4/Predictions/models/%s/frames/%s" % (self._id, test_data.frame_id),
                               data={"deep_features_hidden_layer": layer}), "deepfeatures")
        else:
            j = H2OJob(h2o.api("POST /4/Predictions/models/%s/frames/%s" % (self._id, test_data.frame_id),
                               data={"deep_features_hidden_layer_name": layer}), "deepfeatures")
        j.poll()
        return h2o.get_frame(j.dest_key)

Response: Create a deep feature space on a hidden layer.


Instruction:
Create a Python 3 function for
returning the H2OFrame for the respective weight matrix.

Response:
def weights(self, matrix_id=0):
        """
        Return the frame for the respective weight matrix.

        :param: matrix_id: an integer, ranging from 0 to number of layers, that specifies the weight matrix to return.

        :returns: an H2OFrame which represents the weight matrix identified by matrix_id
        """
        num_weight_matrices = len(self._model_json["output"]["weights"])
        if matrix_id not in list(range(num_weight_matrices)):
            raise ValueError(
                "Weight matrix does not exist. Model has {0} weight matrices (0-based indexing), but matrix {1} "
                "was requested.".format(num_weight_matrices, matrix_id))
        return h2o.get_frame(self._model_json["output"]["weights"][matrix_id]["URL"].split("/")[3])


Instruction:
Write a Python 3 function for
returning the frame for the respective bias vector.

Response:
def biases(self, vector_id=0):
        """
        Return the frame for the respective bias vector.

        :param: vector_id: an integer, ranging from 0 to number of layers, that specifies the bias vector to return.

        :returns: an H2OFrame which represents the bias vector identified by vector_id
        """
        num_bias_vectors = len(self._model_json["output"]["biases"])
        if vector_id not in list(range(num_bias_vectors)):
            raise ValueError(
                "Bias vector does not exist. Model has {0} bias vectors (0-based indexing), but vector {1} "
                "was requested.".format(num_bias_vectors, vector_id))
        return h2o.get_frame(self._model_json["output"]["biases"][vector_id]["URL"].split("/")[3])


Instruction:
Explain what the following Python 3 code does
def model_performance(self, test_data=None, train=False, valid=False, xval=False):
        if test_data is None:
            if not train and not valid and not xval: train = True  # default to train
            if train: return self._model_json["output"]["training_metrics"]
            if valid: return self._model_json["output"]["validation_metrics"]
            if xval: return self._model_json["output"]["cross_validation_metrics"]

        else:  # cases dealing with test_data not None
            if not isinstance(test_data, h2o.H2OFrame):
                raise ValueError("`test_data` must be of type H2OFrame.  Got: " + type(test_data))
            if (self._model_json["response_column_name"] != None) and not(self._model_json["response_column_name"] in test_data.names):
                print("WARNING: Model metrics cannot be calculated and metric_json is empty due to the absence of the response column in your dataset.")
                return
            res = h2o.api("POST /3/ModelMetrics/models/%s/frames/%s" % (self.model_id, test_data.frame_id))

            # FIXME need to do the client-side filtering...  (PUBDEV-874)
            raw_metrics = None
            for mm in res["model_metrics"]:
                if mm["frame"] is not None and mm["frame"]["name"] == test_data.frame_id:
                    raw_metrics = mm
                    break
            return self._metrics_class(raw_metrics, algo=self._model_json["algo"])

Response: Generate model metrics for this model on a given test_data.


Instruction:
Given the following Python 3 function, write the documentation
def scoring_history(self):
        model = self._model_json["output"]
        if "scoring_history" in model and model["scoring_history"] is not None:
            return model["scoring_history"].as_data_frame()
        print("No score history for this model")

Response: Retrieve Model Score History.


Instruction:
Create a Python 3 function for
printing innards of model without regards to type.

Response:
def show(self):
        """Print innards of model, without regards to type."""
        if self._future:
            self._job.poll_once()
            return
        if self._model_json is None:
            print("No model trained yet")
            return
        if self.model_id is None:
            print("This H2OEstimator has been removed.")
            return
        model = self._model_json["output"]
        print("Model Details")
        print("=============")

        print(self.__class__.__name__, ": ", self._model_json["algo_full_name"])
        print("Model Key: ", self._id)

        self.summary()

        print()
        # training metrics
        tm = model["training_metrics"]
        if tm: tm.show()
        vm = model["validation_metrics"]
        if vm: vm.show()
        xm = model["cross_validation_metrics"]
        if xm: xm.show()
        xms = model["cross_validation_metrics_summary"]
        if xms: xms.show()

        if "scoring_history" in model and model["scoring_history"]:
            model["scoring_history"].show()
        if "variable_importances" in model and model["variable_importances"]:
            model["variable_importances"].show()


Instruction:
Can you tell what is the following Python 3 function doing
def varimp(self, use_pandas=False):
        model = self._model_json["output"]
        if self.algo=='glm' or "variable_importances" in list(model.keys()) and model["variable_importances"]:
            if self.algo=='glm':
                tempvals = model["standardized_coefficient_magnitudes"].cell_values
                maxVal = 0
                sum=0
                for item in tempvals:
                    sum=sum+item[1]
                    if item[1]>maxVal:
                        maxVal = item[1]
                vals = []
                for item in tempvals:
                    tempT = (item[0], item[1], item[1]/maxVal, item[1]/sum)
                    vals.append(tempT)
                header = ["variable", "relative_importance", "scaled_importance", "percentage"]
            else:
                vals = model["variable_importances"].cell_values
                header = model["variable_importances"].col_header

            if use_pandas and can_use_pandas():
                import pandas
                return pandas.DataFrame(vals, columns=header)
            else:
                return vals
        else:
            print("Warning: This model doesn't have variable importances")

Response: Pretty print the variable importances or return them in a list or Pandas DataFrame.


Instruction:
Implement a function in Python 3 to
return the residual degress of freedom for the current set.

Response:
def residual_degrees_of_freedom(self, train=False, valid=False, xval=False):
        """
        Retreive the residual degress of freedom if this model has the attribute, or None otherwise.

        :param bool train: Get the residual dof for the training set. If both train and valid are False, then train
            is selected by default.
        :param bool valid: Get the residual dof for the validation set. If both train and valid are True, then train
            is selected by default.

        :returns: Return the residual dof, or None if it is not present.
        """
        if xval: raise H2OValueError("Cross-validation metrics are not available.")
        if not train and not valid: train = True
        if train and valid:         train = True
        if train:
            return self._model_json["output"]["training_metrics"].residual_degrees_of_freedom()
        else:
            return self._model_json["output"]["validation_metrics"].residual_degrees_of_freedom()


Instruction:
How would you explain what the following Python 3 function does
def coef(self):
        tbl = self._model_json["output"]["coefficients_table"]
        if tbl is None:
            return None
        return {name: coef for name, coef in zip(tbl["names"], tbl["coefficients"])}

Response: Return the coefficients which can be applied to the non - standardized data.


Instruction:
Given the following Python 3 function, write the documentation
def coef_norm(self):
        if self._model_json["output"]["model_category"]=="Multinomial":
            tbl = self._model_json["output"]["standardized_coefficient_magnitudes"]
            if tbl is None:
                return None
            return {name: coef for name, coef in zip(tbl["names"], tbl["coefficients"])}
        else:
            tbl = self._model_json["output"]["coefficients_table"]
            if tbl is None:
                return None
            return {name: coef for name, coef in zip(tbl["names"], tbl["standardized_coefficients"])}

Response: Return coefficients fitted on the standardized data.


Instruction:
Write a Python 3 script for
downloading the POJO for this model to the specified path.

Response:
def download_pojo(self, path="", get_genmodel_jar=False, genmodel_name=""):
        """
        Download the POJO for this model to the directory specified by path.

        If path is an empty string, then dump the output to screen.

        :param path:  An absolute path to the directory where POJO should be saved.
        :param get_genmodel_jar: if True, then also download h2o-genmodel.jar and store it in folder ``path``.
        :param genmodel_name Custom name of genmodel jar
        :returns: name of the POJO file written.
        """
        assert_is_type(path, str)
        assert_is_type(get_genmodel_jar, bool)
        path = path.rstrip("/")
        return h2o.download_pojo(self, path, get_jar=get_genmodel_jar, jar_name=genmodel_name)


Instruction:
Given the following Python 3 function, write the documentation
def download_mojo(self, path=".", get_genmodel_jar=False, genmodel_name=""):
        assert_is_type(path, str)
        assert_is_type(get_genmodel_jar, bool)

        if not self.have_mojo:
            raise H2OValueError("Export to MOJO not supported")

        if get_genmodel_jar:
            if genmodel_name == "":
                h2o.api("GET /3/h2o-genmodel.jar", save_to=os.path.join(path, "h2o-genmodel.jar"))
            else:
                h2o.api("GET /3/h2o-genmodel.jar", save_to=os.path.join(path, genmodel_name))
        return h2o.api("GET /3/Models/%s/mojo" % self.model_id, save_to=path)

Response: Download the MOJO file in MOJO format.


Instruction:
Explain what the following Python 3 code does
def save_mojo(self, path="", force=False):
        assert_is_type(path, str)
        assert_is_type(force, bool)
        if not self.have_mojo:
            raise H2OValueError("Export to MOJO not supported")
        path = os.path.join(os.getcwd() if path == "" else path, self.model_id + ".zip")
        return h2o.api("GET /99/Models.mojo/%s" % self.model_id, data={"dir": path, "force": force})["dir"]

Response: Save the current state of the current model as a MOJO.


Instruction:
How would you implement a function in Python 3 that
saves the details of the current model in JSON Format to disk.

Response:
def save_model_details(self, path="", force=False):
        """
        Save Model Details of an H2O Model in JSON Format to disk.

        :param model: The model object to save.
        :param path: a path to save the model details at (hdfs, s3, local)
        :param force: if True overwrite destination directory in case it exists, or throw exception if set to False.

        :returns str: the path of the saved model details
        """
        assert_is_type(path, str)
        assert_is_type(force, bool)
        path = os.path.join(os.getcwd() if path == "" else path, self.model_id + ".json")
        return h2o.api("GET /99/Models/%s/json" % self.model_id, data={"dir": path, "force": force})["dir"]


Instruction:
How would you explain what the following Python 3 function does
def partial_plot(self, data, cols, destination_key=None, nbins=20, weight_column=None,
                     plot=True, plot_stddev = True, figsize=(7, 10), server=False, include_na=False, user_splits=None,
                     save_to_file=None):
        """
        Create partial dependence plot which gives a graphical depiction of the marginal effect of a variable on the
        response. The effect of a variable is measured in change in the mean response.

        :param H2OFrame data: An H2OFrame object used for scoring and constructing the plot.
        :param cols: Feature(s) for which partial dependence will be calculated.
        :param destination_key: An key reference to the created partial dependence tables in H2O.
        :param nbins: Number of bins used. For categorical columns make sure the number of bins exceed the level count. If you enable add_missing_NA, the returned length will be nbin+1.
        :param weight_column: A string denoting which column of data should be used as the weight column.
        :param plot: A boolean specifying whether to plot partial dependence table.
        :param plot_stddev: A boolean specifying whether to add std err to partial dependence plot.
        :param figsize: Dimension/size of the returning plots, adjust to fit your output cells.
        :param server: ?
        :param include_na: A boolean specifying whether missing value should be included in the Feature values.
        :param user_splits: a dictionary containing column names as key and user defined split values as value in a list.
        :param save_to_file Fully qualified name to an image file the resulting plot should be saved to, e.g. '/home/user/pdpplot.png'. The 'png' postfix might be omitted. If the file already exists, it will be overridden. Plot is only saved if plot = True.
        :returns: Plot and list of calculated mean response tables for each feature requested.
        """

        if not isinstance(data, h2o.H2OFrame): raise ValueError("data must be an instance of H2OFrame")
        assert_is_type(cols, [str])
        assert_is_type(destination_key, None, str)
        assert_is_type(nbins, int)
        assert_is_type(plot, bool)
        assert_is_type(figsize, (int, int))

        # Check cols specified exist in frame data
        for xi in cols:
            if xi not in data.names:
                raise H2OValueError("Column %s does not exist in the training frame" % xi)
        if isinstance(weight_column, int) and not (weight_column == -1):
            raise H2OValueError("Weight column should be a column name in your data frame.")
        elif isinstance(weight_column, str): # index is a name
            if weight_column not in data.names:
                raise H2OValueError("Column %s does not exist in the data frame" % weight_column)
            weight_column = data.names.index(weight_column)

        kwargs = {}
        kwargs["cols"] = cols
        kwargs["model_id"] = self.model_id
        kwargs["frame_id"] = data.frame_id
        kwargs["nbins"] = nbins
        kwargs["destination_key"] = destination_key
        kwargs["weight_column_index"] = weight_column
        kwargs["add_missing_na"] = include_na

        # extract user defined split points from dict user_splits into an integer array of column indices
        # and a double array of user define values for the corresponding columns
        if not(user_splits == None):
            if not(isinstance(user_splits, dict)):
                raise H2OValueError("user_splits must be a Python dict.")
            else:
                if len(user_splits)>0: # do nothing with an empty dict
                    user_cols = []
                    user_values = []
                    user_num_splits = []
                    data_ncol = data.ncol
                    column_names = data.names
                    for colKey,val in user_splits.items():
                        if is_type(colKey, str) and colKey in column_names:
                            user_cols.append(colKey)
                        elif isinstance(colKey, int) and colKey < data_ncol:
                            user_cols.append(column_names[colKey])
                        else:
                            raise H2OValueError("column names/indices used in user_splits are not valid.  They "
                                                "should be chosen from the columns of your data set.")

                        if data[colKey].isfactor()[0] or data[colKey].isnumeric()[0]: # replace enum string with actual value
                            nVal = len(val)
                            if data[colKey].isfactor()[0]:
                                domains = data[colKey].levels()[0]

                                numVal = [0]*nVal
                                for ind in range(nVal):
                                    if (val[ind] in domains):
                                        numVal[ind] = domains.index(val[ind])
                                    else:
                                        raise H2OValueError("Illegal enum value {0} encountered.  To include missing"
                                                        " values in your feature values, set include_na to "
                                                        "True".format(val[ind]))

                                user_values.extend(numVal)
                            else:
                                user_values.extend(val)
                            user_num_splits.append(nVal)
                        else:
                            raise H2OValueError("Partial dependency plots are generated for numerical and categorical "
                                                "columns only.")
                    kwargs["user_cols"] = user_cols
                    kwargs["user_splits"] = user_values
                    kwargs["num_user_splits"] = user_num_splits
                else:
                    kwargs["user_cols"] = None
                    kwargs["user_splits"] = None
                    kwargs["num_user_splits"] = None

        json = H2OJob(h2o.api("POST /3/PartialDependence/", data=kwargs),  job_type="PartialDependencePlot").poll()
        json = h2o.api("GET /3/PartialDependence/%s" % json.dest_key)

        # Extract partial dependence data from json response
        pps = json["partial_dependence_data"]

        # Plot partial dependence plots using matplotlib
        if plot:
            plt = _get_matplotlib_pyplot(server)
            if not plt: return

            fig, axs = plt.subplots(len(cols), squeeze=False, figsize=figsize)
            for i, pp in enumerate(pps):
                # Check weather column was categorical or numeric
                col = cols[i]
                cat = data[col].isfactor()[0]
                upper = [a + b for a, b in zip(pp[1], pp[2]) ]
                lower = [a - b for a, b in zip(pp[1], pp[2]) ]
                if cat:
                    labels = pp[0]
                    x = range(len(labels))
                    y = pp[1]
                    axs[i, 0].plot(x, y, "ro")
                    if plot_stddev:
                        axs[i, 0].plot(x, lower, 'b--')
                        axs[i, 0].plot(x, upper, 'b--')
                    axs[i, 0].set_ylim(min(lower) - 0.1*abs(min(lower)), max(upper) + 0.1*abs(max(upper)))
                    axs[i, 0].set_xticks(x)
                    axs[i, 0].set_xticklabels(labels)
                    axs[i, 0].margins(0.2)
                else:
                    x = pp[0]
                    y = pp[1]
                    axs[i, 0].plot(x, y, "r-")
                    if plot_stddev:
                        axs[i, 0].plot(x, lower, 'b--')
                        axs[i, 0].plot(x, upper, 'b--')
                    axs[i, 0].set_xlim(min(x), max(x))
                    axs[i, 0].set_ylim(min(lower) - 0.1*abs(min(lower)), max(upper) + 0.1*abs(max(upper)))

                axs[i, 0].set_title("Partial Dependence Plot For {}".format(col))
                axs[i, 0].set_xlabel(pp.col_header[0])
                axs[i, 0].set_ylabel(pp.col_header[1])
                axs[i, 0].xaxis.grid()
                axs[i, 0].yaxis.grid()
            if len(col) > 1:
                fig.tight_layout(pad=0.4, w_pad=0.5, h_pad=1.0)
            if(save_to_file is not None):
                plt.savefig(save_to_file)

        return pps

Response: Create a partial dependence plot for each feature requested.


Instruction:
Can you tell what is the following Python 3 function doing
def varimp_plot(self, num_of_features=None, server=False):
        assert_is_type(num_of_features, None, int)
        assert_is_type(server, bool)

        plt = _get_matplotlib_pyplot(server)
        if not plt: return

        # get the variable importances as a list of tuples, do not use pandas dataframe
        importances = self.varimp(use_pandas=False)
        # features labels correspond to the first value of each tuple in the importances list
        feature_labels = [tup[0] for tup in importances]
        # relative importances correspond to the first value of each tuple in the importances list
        scaled_importances = [tup[2] for tup in importances]
        # specify bar centers on the y axis, but flip the order so largest bar appears at top
        pos = range(len(feature_labels))[::-1]
        # specify the bar lengths
        val = scaled_importances

        # # check that num_of_features is an integer
        # if num_of_features is None:
        #     num_of_features = len(val)

        # default to 10 or less features if num_of_features is not specified
        if num_of_features is None:
            num_of_features = min(len(val), 10)

        fig, ax = plt.subplots(1, 1, figsize=(14, 10))
        # create separate plot for the case where num_of_features == 1
        if num_of_features == 1:
            plt.barh(pos[0:num_of_features], val[0:num_of_features], align="center",
                     height=0.8, color="#1F77B4", edgecolor="none")
            # Hide the right and top spines, color others grey
            ax.spines["right"].set_visible(False)
            ax.spines["top"].set_visible(False)
            ax.spines["bottom"].set_color("#7B7B7B")
            ax.spines["left"].set_color("#7B7B7B")
            # Only show ticks on the left and bottom spines
            ax.yaxis.set_ticks_position("left")
            ax.xaxis.set_ticks_position("bottom")
            plt.yticks(pos[0:num_of_features], feature_labels[0:num_of_features])
            ax.margins(None, 0.5)

        else:
            plt.barh(pos[0:num_of_features], val[0:num_of_features], align="center",
                     height=0.8, color="#1F77B4", edgecolor="none")
            # Hide the right and top spines, color others grey
            ax.spines["right"].set_visible(False)
            ax.spines["top"].set_visible(False)
            ax.spines["bottom"].set_color("#7B7B7B")
            ax.spines["left"].set_color("#7B7B7B")
            # Only show ticks on the left and bottom spines
            ax.yaxis.set_ticks_position("left")
            ax.xaxis.set_ticks_position("bottom")
            plt.yticks(pos[0:num_of_features], feature_labels[0:num_of_features])
            plt.ylim([min(pos[0:num_of_features])- 1, max(pos[0:num_of_features])+1])
            # ax.margins(y=0.5)

        # check which algorithm was used to select right plot title
        if self._model_json["algo"] == "gbm":
            plt.title("Variable Importance: H2O GBM", fontsize=20)
            if not server: plt.show()
        elif self._model_json["algo"] == "drf":
            plt.title("Variable Importance: H2O DRF", fontsize=20)
            if not server: plt.show()
        elif self._model_json["algo"] == "xgboost":
            plt.title("Variable Importance: H2O XGBoost", fontsize=20)
            if not server: plt.show()
        # if H2ODeepLearningEstimator has variable_importances == True
        elif self._model_json["algo"] == "deeplearning":
            plt.title("Variable Importance: H2O Deep Learning", fontsize=20)
            if not server: plt.show()
        elif self._model_json["algo"] == "glm":
            plt.title("Variable Importance: H2O GLM", fontsize=20)
            if not server: plt.show()
        else:
            raise H2OValueError("A variable importances plot is not implemented for this type of model")

Response: Plots the variable importance for a trained model.


Instruction:
Implement a Python 3 function for
plotting a GLM model s standardized coefficient magnitudes.

Response:
def std_coef_plot(self, num_of_features=None, server=False):
        """
        Plot a GLM model"s standardized coefficient magnitudes.

        :param num_of_features: the number of features shown in the plot.
        :param server: ?

        :returns: None.
        """
        assert_is_type(num_of_features, None, I(int, lambda x: x > 0))

        # check that model is a glm
        if self._model_json["algo"] != "glm":
            raise H2OValueError("This function is available for GLM models only")

        plt = _get_matplotlib_pyplot(server)
        if not plt: return

        # get unsorted tuple of labels and coefficients
        unsorted_norm_coef = self.coef_norm().items()
        # drop intercept value then sort tuples by the coefficient"s absolute value
        drop_intercept = [tup for tup in unsorted_norm_coef if tup[0] != "Intercept"]
        norm_coef = sorted(drop_intercept, key=lambda x: abs(x[1]), reverse=True)

        signage = []
        for element in norm_coef:
            # if positive including zero, color blue, else color orange (use same colors as Flow)
            if element[1] >= 0:
                signage.append("#1F77B4")  # blue
            else:
                signage.append("#FF7F0E")  # dark orange

        # get feature labels and their corresponding magnitudes
        feature_labels = [tup[0] for tup in norm_coef]
        norm_coef_magn = [abs(tup[1]) for tup in norm_coef]
        # specify bar centers on the y axis, but flip the order so largest bar appears at top
        pos = range(len(feature_labels))[::-1]
        # specify the bar lengths
        val = norm_coef_magn

        # check number of features, default is all the features
        if num_of_features is None:
            num_of_features = len(val)

        # plot horizontal plot
        fig, ax = plt.subplots(1, 1, figsize=(14, 10))
        # create separate plot for the case where num_of_features = 1
        if num_of_features == 1:
            plt.barh(pos[0], val[0],
                     align="center", height=0.8, color=signage[0], edgecolor="none")
            # Hide the right and top spines, color others grey
            ax.spines["right"].set_visible(False)
            ax.spines["top"].set_visible(False)
            ax.spines["bottom"].set_color("#7B7B7B")
            ax.spines["left"].set_color("#7B7B7B")
            # Only show ticks on the left and bottom spines
            ax.yaxis.set_ticks_position("left")
            ax.xaxis.set_ticks_position("bottom")
            plt.yticks([0], feature_labels[0])
            ax.margins(None, 0.5)

        else:
            plt.barh(pos[0:num_of_features], val[0:num_of_features],
                     align="center", height=0.8, color=signage[0:num_of_features], edgecolor="none")
            # Hide the right and top spines, color others grey
            ax.spines["right"].set_visible(False)
            ax.spines["top"].set_visible(False)
            ax.spines["bottom"].set_color("#7B7B7B")
            ax.spines["left"].set_color("#7B7B7B")
            # Only show ticks on the left and bottom spines
            ax.yaxis.set_ticks_position("left")
            ax.xaxis.set_ticks_position("bottom")
            plt.yticks(pos[0:num_of_features], feature_labels[0:num_of_features])
            ax.margins(None, 0.05)

        # generate custom fake lines that will be used as legend entries:
        # check if positive and negative values exist
        # if positive create positive legend
        if "#1F77B4" in signage[0:num_of_features] and "#FF7F0E" not in signage[0:num_of_features]:
            color_ids = ("Positive",)
            markers = [plt.Line2D([0, 0], [0, 0], color=color, marker="s", linestyle="")
                       for color in signage[0:num_of_features]]
            lgnd = plt.legend(markers, color_ids, numpoints=1, loc="best", frameon=False, fontsize=13)
            lgnd.legendHandles[0]._legmarker.set_markersize(10)
        # if neg create neg legend
        elif "#FF7F0E" in signage[0:num_of_features] and "#1F77B4" not in signage[0:num_of_features]:
            color_ids = ("Negative",)
            markers = [plt.Line2D([0, 0], [0, 0], color=color, marker="s", linestyle="")
                       for color in set(signage[0:num_of_features])]
            lgnd = plt.legend(markers, color_ids, numpoints=1, loc="best", frameon=False, fontsize=13)
            lgnd.legendHandles[0]._legmarker.set_markersize(10)
        # if both provide both colors in legend
        else:
            color_ids = ("Positive", "Negative")
            markers = [plt.Line2D([0, 0], [0, 0], color=color, marker="s", linestyle="")
                       for color in ['#1F77B4', '#FF7F0E']] # blue should always be positive, orange negative
            lgnd = plt.legend(markers, color_ids, numpoints=1, loc="best", frameon=False, fontsize=13)
            lgnd.legendHandles[0]._legmarker.set_markersize(10)
            lgnd.legendHandles[1]._legmarker.set_markersize(10)

        # Hide the right and top spines, color others grey
        ax.spines["right"].set_visible(False)
        ax.spines["top"].set_visible(False)
        ax.spines["bottom"].set_color("#7B7B7B")
        ax.spines["left"].set_color("#7B7B7B")

        # Only show ticks on the left and bottom spines
        # ax.yaxis.set_ticks_position("left")
        # ax.xaxis.set_ticks_position("bottom")
        plt.yticks(pos[0:num_of_features], feature_labels[0:num_of_features])
        plt.tick_params(axis="x", which="minor", bottom="off", top="off",  labelbottom="off")
        plt.title("Standardized Coef. Magnitudes: H2O GLM", fontsize=20)
        # plt.axis("tight")
        # show plot
        if not server: plt.show()


Instruction:
Create a Python 3 function to
check that the two sets of records are the same length.

Response:
def _check_targets(y_actual, y_predicted):
        """Check that y_actual and y_predicted have the same length.

        :param H2OFrame y_actual:
        :param H2OFrame y_predicted:

        :returns: None
        """
        if len(y_actual) != len(y_predicted):
            raise ValueError("Row mismatch: [{},{}]".format(len(y_actual), len(y_predicted)))


Instruction:
Explain what the following Python 3 code does
def cross_validation_models(self):
        cvmodels = self._model_json["output"]["cross_validation_models"]
        if cvmodels is None: return None
        m = []
        for p in cvmodels: m.append(h2o.get_model(p["name"]))
        return m

Response: Obtain a list of cross - validation models.


Instruction:
Create a Python 3 function for
obtaining the H2OFrame objects representing the cross - validation predictions on the training data.

Response:
def cross_validation_predictions(self):
        """
        Obtain the (out-of-sample) holdout predictions of all cross-validation models on their holdout data.

        Note that the predictions are expanded to the full number of rows of the training data, with 0 fill-in.

        :returns: list of H2OFrame objects.
        """
        preds = self._model_json["output"]["cross_validation_predictions"]
        if preds is None: return None
        m = []
        for p in preds: m.append(h2o.get_frame(p["name"]))
        return m


Instruction:
Can you generate the documentation for the following Python 3 function
def _run_demo(body_fn, interactive, echo, testing):
    import colorama
    from colorama import Style, Fore
    colorama.init()

    class StopExecution(Exception):
        """Helper class for cancelling the demo."""

    assert_is_type(body_fn, type(_run_demo))
    # Reformat description by removing extra spaces; then print it.
    if body_fn.__doc__:
        desc_lines = body_fn.__doc__.split("\n")
        while desc_lines[0].strip() == "":
            desc_lines = desc_lines[1:]
        while desc_lines[-1].strip() == "":
            desc_lines = desc_lines[:-1]
        strip_spaces = min(len(line) - len(line.lstrip(" ")) for line in desc_lines[1:] if line.strip() != "")
        maxlen = max(len(line) for line in desc_lines)
        print(Fore.CYAN)
        print("-" * maxlen)
        for line in desc_lines:
            print(line[strip_spaces:].rstrip())
        print("-" * maxlen)
        print(Style.RESET_ALL, end="")

    # Prepare the executor function
    def controller():
        """Print to console the next block of commands, and wait for keypress."""
        try:
            raise RuntimeError("Catch me!")
        except RuntimeError:
            print()
            # Extract and print lines that will be executed next
            if echo:
                tb = sys.exc_info()[2]
                fr = tb.tb_frame.f_back
                filename = fr.f_code.co_filename
                linecache.checkcache(filename)
                line = linecache.getline(filename, fr.f_lineno, fr.f_globals).rstrip()
                indent_len = len(line) - len(line.lstrip(" "))
                assert line[indent_len:] == "go()"
                i = fr.f_lineno
                output_lines = []
                n_blank_lines = 0
                while True:
                    i += 1
                    line = linecache.getline(filename, i, fr.f_globals).rstrip()
                    # Detect dedent
                    if line[:indent_len].strip() != "": break
                    line = line[indent_len:]
                    if line == "go()": break
                    style = Fore.LIGHTBLACK_EX if line.lstrip().startswith("#") else Style.BRIGHT
                    prompt = "... " if line.startswith(" ") else ">>> "
                    output_lines.append(Fore.CYAN + prompt + Fore.RESET + style + line + Style.RESET_ALL)
                    del style  # Otherwise exception print-outs may get messed-up...
                    if line.strip() == "":
                        n_blank_lines += 1
                        if n_blank_lines > 5: break  # Just in case we hit file end or something
                    else:
                        n_blank_lines = 0
                for line in output_lines[:-n_blank_lines]:
                    print(line)

            # Prompt for user input
            if interactive:
                print("\n" + Style.DIM + "(press any key)" + Style.RESET_ALL, end="")
                key = _wait_for_keypress()
                print("\r                     \r", end="")
                if key.lower() == "q":
                    raise StopExecution()

    # Replace h2o.init() with a stub when running in "test" mode
    _h2o_init = h2o.init
    if testing:
        h2o.init = lambda *args, **kwargs: None

    # Run the test
    try:
        body_fn(controller)
        print("\n" + Fore.CYAN + "---- End of Demo ----" + Style.RESET_ALL)
    except (StopExecution, KeyboardInterrupt):
        print("\n" + Fore.RED + "---- Demo aborted ----" + Style.RESET_ALL)

    # Clean-up
    if testing:
        h2o.init = _h2o_init
    print()
    colorama.deinit()

Response: Runs a demo function.


Instruction:
Write a Python 3 script for
waiting for a key press on the console and return it.

Response:
def _wait_for_keypress():
    """
    Wait for a key press on the console and return it.

    Borrowed from http://stackoverflow.com/questions/983354/how-do-i-make-python-to-wait-for-a-pressed-key
    """
    result = None
    if os.name == "nt":
        # noinspection PyUnresolvedReferences
        import msvcrt
        result = msvcrt.getch()
    else:
        import termios
        fd = sys.stdin.fileno()

        oldterm = termios.tcgetattr(fd)
        newattr = termios.tcgetattr(fd)
        newattr[3] = newattr[3] & ~termios.ICANON & ~termios.ECHO
        termios.tcsetattr(fd, termios.TCSANOW, newattr)

        try:
            result = sys.stdin.read(1)
        except IOError:
            pass
        finally:
            termios.tcsetattr(fd, termios.TCSAFLUSH, oldterm)

    return result


Instruction:
Can you generate the documentation for the following Python 3 function
def make(keyvals):
        kwargs = {}
        for key, value in keyvals:
            if key == "columns":
                kwargs["col_formats"] = [c["format"] for c in value]
                kwargs["col_types"] = [c["type"] for c in value]
                kwargs["col_header"] = [c["name"] for c in value]
                kwargs["row_header"] = len(value)
            if key == "name": kwargs["table_header"] = value
            if key == "description": kwargs["table_description"] = value
            if key == "data": kwargs["raw_cell_values"] = value
        return H2OTwoDimTable(**kwargs)

Response: Create a new H2ODimTable object from a list of key - value tuples which are a pre - cursor to JSON dict.


Instruction:
How would you explain what the following Python 3 function does
def as_data_frame(self):
        if can_use_pandas():
            import pandas
            pandas.options.display.max_colwidth = 70
            return pandas.DataFrame(self._cell_values, columns=self._col_header)
        return self

Response: Convert to a python data frame.


Instruction:
Make a summary of the following Python 3 code
def show(self, header=True):
        # if h2o.can_use_pandas():
        #  import pandas
        #  pandas.options.display.max_rows = 20
        #  print pandas.DataFrame(self._cell_values,columns=self._col_header)
        #  return
        if header and self._table_header:
            print(self._table_header + ":", end=' ')
            if self._table_description: print(self._table_description)
        print()
        table = copy.deepcopy(self._cell_values)
        nr = 0
        if _is_list_of_lists(table): nr = len(
            table)  # only set if we truly have multiple rows... not just one long row :)
        if nr > 20:  # create a truncated view of the table, first/last 5 rows
            trunc_table = []
            trunc_table += [v for v in table[:5]]
            trunc_table.append(["---"] * len(table[0]))
            trunc_table += [v for v in table[(nr - 5):]]
            table = trunc_table

        H2ODisplay(table, self._col_header, numalign="left", stralign="left")
        if nr > 20 and can_use_pandas(): print('\nSee the whole table with table.as_data_frame()')

Response: Print the contents of this table.


Instruction:
Explain what the following Python 3 code does
def reader(stream, dialect=DIALECT, encoding=False, **fmtparams):
    if encoding is False:
        return UnicodeTextReader(stream, dialect, **fmtparams)
    if encoding is None:
        encoding = none_encoding()
    if not is_8bit_clean(encoding):
        raise NotImplementedError
    return UnicodeBytesReader(stream, dialect, encoding, **fmtparams)

Response: r CSV reader yielding lists of unicode strings.


Instruction:
Can you implement a function in Python 3 that
starts a new H2O server on the local machine.

Response:
def start(jar_path=None, nthreads=-1, enable_assertions=True, max_mem_size=None, min_mem_size=None,
              ice_root=None, log_dir=None, log_level=None, port="54321+", name=None, extra_classpath=None,
              verbose=True, jvm_custom_args=None, bind_to_localhost=True):
        """
        Start new H2O server on the local machine.

        :param jar_path: Path to the h2o.jar executable. If not given, then we will search for h2o.jar in the
            locations returned by `._jar_paths()`.
        :param nthreads: Number of threads in the thread pool. This should be related to the number of CPUs used.
            -1 means use all CPUs on the host. A positive integer specifies the number of CPUs directly.
        :param enable_assertions: If True, pass `-ea` option to the JVM.
        :param max_mem_size: Maximum heap size (jvm option Xmx), in bytes.
        :param min_mem_size: Minimum heap size (jvm option Xms), in bytes.
        :param log_dir: Directory for H2O logs to be stored if a new instance is started. Default directory is determined
        by H2O internally.
        :param log_level: The logger level for H2O if a new instance is started.
        :param ice_root: A directory where H2O stores its temporary files. Default location is determined by
            tempfile.mkdtemp().
        :param port: Port where to start the new server. This could be either an integer, or a string of the form
            "DDDDD+", indicating that the server should start looking for an open port starting from DDDDD and up.
        :param name: name of the h2o cluster to be started
        :param extra_classpath List of paths to libraries that should be included on the Java classpath.
        :param verbose: If True, then connection info will be printed to the stdout.
        :param jvm_custom_args Custom, user-defined arguments for the JVM H2O is instantiated in
        :param bind_to_localhost A flag indicating whether access to the H2O instance should be restricted to the local
            machine (default) or if it can be reached from other computers on the network.
            Only applicable when H2O is started from the Python client.

        :returns: a new H2OLocalServer instance
        """
        assert_is_type(jar_path, None, str)
        assert_is_type(port, None, int, str)
        assert_is_type(name, None, str)
        assert_is_type(nthreads, -1, BoundInt(1, 4096))
        assert_is_type(enable_assertions, bool)
        assert_is_type(min_mem_size, None, int)
        assert_is_type(max_mem_size, None, BoundInt(1 << 25))
        assert_is_type(log_dir, str, None)
        assert_is_type(log_level, str, None)
        assert_satisfies(log_level, log_level in [None, "TRACE", "DEBUG", "INFO", "WARN", "ERRR", "FATA"])
        assert_is_type(ice_root, None, I(str, os.path.isdir))
        assert_is_type(extra_classpath, None, [str])
        assert_is_type(jvm_custom_args, list, None)
        assert_is_type(bind_to_localhost, bool)
        if jar_path:
            assert_satisfies(jar_path, jar_path.endswith("h2o.jar"))

        if min_mem_size is not None and max_mem_size is not None and min_mem_size > max_mem_size:
            raise H2OValueError("`min_mem_size`=%d is larger than the `max_mem_size`=%d" % (min_mem_size, max_mem_size))
        if port is None: port = "54321+"
        baseport = None
        # TODO: get rid of this port gimmick and have 2 separate parameters.
        if is_type(port, str):
            if port.isdigit():
                port = int(port)
            else:
                if not(port[-1] == "+" and port[:-1].isdigit()):
                    raise H2OValueError("`port` should be of the form 'DDDD+', where D is a digit. Got: %s" % port)
                baseport = int(port[:-1])
                port = 0

        hs = H2OLocalServer()
        hs._verbose = bool(verbose)
        hs._jar_path = hs._find_jar(jar_path)
        hs._extra_classpath = extra_classpath
        hs._ice_root = ice_root
        hs._name = name
        if not ice_root:
            hs._ice_root = tempfile.mkdtemp()
            hs._tempdir = hs._ice_root

        if verbose: print("Attempting to start a local H2O server...")
        hs._launch_server(port=port, baseport=baseport, nthreads=int(nthreads), ea=enable_assertions,
                          mmax=max_mem_size, mmin=min_mem_size, jvm_custom_args=jvm_custom_args,
                          bind_to_localhost=bind_to_localhost, log_dir=log_dir, log_level=log_level)
        if verbose: print("  Server is running at %s://%s:%d" % (hs.scheme, hs.ip, hs.port))
        atexit.register(lambda: hs.shutdown())
        return hs


Instruction:
Given the following Python 3 function, write the documentation
def shutdown(self):
        if not self._process: return
        try:
            kill_time = time.time() + self._TIME_TO_KILL
            while self._process.poll() is None and time.time() < kill_time:
                self._process.terminate()
                time.sleep(0.2)
            if self._process().poll() is None:
                self._process.kill()
                time.sleep(0.2)
            if self._verbose:
                print("Local H2O server %s:%s stopped." % (self.ip, self.port))
        except:
            pass
        self._process = None

Response: Shut down the server by trying to terminate the server process gracefully.


Instruction:
How would you explain what the following Python 3 function does
def _find_jar(self, path0=None):
        jar_paths = [path0] if path0 else self._jar_paths()
        searched_paths = []
        for jp in jar_paths:
            searched_paths.append(jp)
            if os.path.exists(jp):
                return jp
        raise H2OStartupError("Cannot start local server: h2o.jar not found. Paths searched:\n" +
                              "".join("    %s\n" % s for s in searched_paths))

Response: Find the location of an h2o. jar executable.


Instruction:
Can you create a Python 3 function that
produces potential paths for an h2o. jar executable.

Response:
def _jar_paths():
        """Produce potential paths for an h2o.jar executable."""

        # PUBDEV-3534 hook to use arbitrary h2o.jar
        own_jar = os.getenv("H2O_JAR_PATH", "")
        if own_jar != "":
            if not os.path.isfile(own_jar):
                raise H2OStartupError("Environment variable H2O_JAR_PATH is set to '%d' but file does not exists, unset environment variable or provide valid path to h2o.jar file." % own_jar)
            yield own_jar

        # Check if running from an h2o-3 src folder (or any subfolder), in which case use the freshly-built h2o.jar
        cwd_chunks = os.path.abspath(".").split(os.path.sep)
        for i in range(len(cwd_chunks), 0, -1):
            if cwd_chunks[i - 1] == "h2o-3":
                yield os.path.sep.join(cwd_chunks[:i] + ["build", "h2o.jar"])
        # Then check the backend/bin folder:
        # (the following works assuming this code is located in h2o/backend/server.py file)
        backend_dir = os.path.split(os.path.realpath(__file__))[0]
        yield os.path.join(backend_dir, "bin", "h2o.jar")

        # Then try several old locations where h2o.jar might have been installed
        prefix1 = prefix2 = sys.prefix
        # On Unix-like systems Python typically gets installed into /Library/... or /System/Library/... If one of
        # those paths is sys.prefix, then we also build its counterpart.
        if prefix1.startswith(os.path.sep + "Library"):
            prefix2 = os.path.join("", "System", prefix1)
        elif prefix1.startswith(os.path.sep + "System"):
            prefix2 = prefix1[len(os.path.join("", "System")):]
        yield os.path.join(prefix1, "h2o_jar", "h2o.jar")
        yield os.path.join(os.path.abspath(os.sep), "usr", "local", "h2o_jar", "h2o.jar")
        yield os.path.join(prefix1, "local", "h2o_jar", "h2o.jar")
        yield os.path.join(get_config_var("userbase"), "h2o_jar", "h2o.jar")
        yield os.path.join(prefix2, "h2o_jar", "h2o.jar")


Instruction:
Create a Python 3 function to
launch the H2O server.

Response:
def _launch_server(self, port, baseport, mmax, mmin, ea, nthreads, jvm_custom_args, bind_to_localhost, log_dir=None, log_level=None):
        """Actually start the h2o.jar executable (helper method for `.start()`)."""
        self._ip = "127.0.0.1"

        # Find Java and check version. (Note that subprocess.check_output returns the output as a bytes object)
        java = self._find_java()
        self._check_java(java, self._verbose)

        if self._verbose:
            print("  Starting server from " + self._jar_path)
            print("  Ice root: " + self._ice_root)

        # Combine jar path with the optional extra classpath
        classpath = [self._jar_path] if self._extra_classpath is None else [self._jar_path] + self._extra_classpath

        # Construct java command to launch the process
        cmd = [java]

        # ...add JVM options
        cmd += ["-ea"] if ea else []
        for (mq, num) in [("-Xms", mmin), ("-Xmx", mmax)]:
            if num is None: continue
            numstr = "%dG" % (num >> 30) if num == (num >> 30) << 30 else \
                     "%dM" % (num >> 20) if num == (num >> 20) << 20 else \
                     str(num)
            cmd += [mq + numstr]
        if jvm_custom_args is not None:
            for arg in jvm_custom_args:
                assert type(arg) is str
                cmd += [arg]

        cmd += ["-cp", os.pathsep.join(classpath), "water.H2OApp"]  # This should be the last JVM option

        # ...add H2O options
        cmd += ["-ip", self._ip]
        if bind_to_localhost:
            cmd += ["-web_ip", self._ip]
        cmd += ["-port", str(port)] if port else []
        cmd += ["-baseport", str(baseport)] if baseport else []
        cmd += ["-ice_root", self._ice_root]
        cmd += ["-nthreads", str(nthreads)] if nthreads > 0 else []

        if log_dir:
            cmd += ["-log_dir", log_dir]

        if log_level:
            cmd += ["-log_level", log_level]

        if not self._name:
            self._name = "H2O_from_python_%s" % self._tmp_file("salt")
        cmd += ["-name", self._name]

        # Warning: do not change to any higher log-level, otherwise we won't be able to know which port the
        # server is listening to.
        cmd += ["-log_level", "INFO"]

        # Create stdout and stderr files
        self._stdout = self._tmp_file("stdout")
        self._stderr = self._tmp_file("stderr")
        cwd = os.path.abspath(os.getcwd())
        out = open(self._stdout, "w")
        err = open(self._stderr, "w")
        if self._verbose:
            print("  JVM stdout: " + out.name)
            print("  JVM stderr: " + err.name)

        # Launch the process
        win32 = sys.platform == "win32"
        flags = getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0) if win32 else 0
        prex = os.setsid if not win32 else None
        try:
            proc = subprocess.Popen(args=cmd, stdout=out, stderr=err, cwd=cwd, creationflags=flags, preexec_fn=prex)
        except OSError as e:
            traceback = getattr(e, "child_traceback", None)
            raise H2OServerError("Unable to start server: %s" % e, traceback)

        # Wait until the server is up-and-running
        giveup_time = time.time() + self._TIME_TO_START
        while True:
            if proc.poll() is not None:
                raise H2OServerError("Server process terminated with error code %d" % proc.returncode)
            ret = self._get_server_info_from_logs()
            if ret:
                self._scheme = ret[0]
                self._ip = ret[1]
                self._port = ret[2]
                self._process = proc
                break
            if time.time() > giveup_time:
                elapsed_time = time.time() - (giveup_time - self._TIME_TO_START)
                raise H2OServerError("Server wasn't able to start in %f seconds." % elapsed_time)
            time.sleep(0.2)


Instruction:
Here you have a function in Python 3, explain what it does
def _find_java():
        # is java callable directly (doesn't work on windows it seems)?
        java = "java.exe" if sys.platform == "win32" else "java"
        if os.access(java, os.X_OK):
            return java

        # Can Java be found on the PATH?
        for path in os.getenv("PATH").split(os.pathsep):  # not same as os.path.sep!
            full_path = os.path.join(path, java)
            if os.access(full_path, os.X_OK):
                return full_path

        # check if JAVA_HOME is set (for Windows)
        if os.getenv("JAVA_HOME"):
            full_path = os.path.join(os.getenv("JAVA_HOME"), "bin", java)
            if os.path.exists(full_path):
                return full_path

        # check "/Program Files" and "/Program Files (x86)" on Windows
        if sys.platform == "win32":
            # On Windows, backslash on the drive letter is necessary, otherwise os.path.join produces an invalid path
            program_folders = [os.path.join("C:\\", "Program Files", "Java"),
                               os.path.join("C:\\", "Program Files (x86)", "Java"),
                               os.path.join("C:\\", "ProgramData", "Oracle", "Java")]
            for folder in program_folders:
                for dirpath, dirnames, filenames in os.walk(folder):
                    if java in filenames:
                        return os.path.join(dirpath, java)

        # not found...
        raise H2OStartupError("Cannot find Java. Please install the latest JRE from\n"
                              "http://www.oracle.com/technetwork/java/javase/downloads/index.html")

Response: Find location of the Java executable.


Instruction:
How would you explain what the following Python 3 function does
def _tmp_file(self, kind):
        if sys.platform == "win32":
            username = os.getenv("USERNAME")
        else:
            username = os.getenv("USER")
        if not username:
            username = "unknownUser"
        usr = "".join(ch if ch.isalnum() else "_" for ch in username)

        if kind == "salt":
            return usr + "_" + "".join(choice("0123456789abcdefghijklmnopqrstuvwxyz") for _ in range(6))
        else:
            if not self._tempdir:
                self._tempdir = tempfile.mkdtemp()
            return os.path.join(self._tempdir, "h2o_%s_started_from_python.%s" % (usr, kind[3:]))

Response:
Generate a temporary file name for the currently - a
           .


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _get_server_info_from_logs(self):
        searchstr = "INFO: Open H2O Flow in your web browser:"
        with open(self._stdout, "rt") as f:
            for line in f:
                if searchstr in line:
                    url = line[line.index(searchstr) + len(searchstr):].strip().rstrip("/")
                    parts = url.split(":")
                    assert len(parts) == 3 and (parts[0] == "http" or parts[1] == "https") and parts[2].isdigit(), \
                        "Unexpected URL: %s" % url
                    return parts[0], parts[1][2:], int(parts[2])
        return None

Response: This method checks the server s output log and returns the scheme IP and port tuple. If the server has already started None is returned.


Instruction:
Write a Python 3 function for
returning a confusion matrix based of H2O s default prediction threshold for a dataset.

Response:
def confusion_matrix(self, data):
        """
        Returns a confusion matrix based of H2O's default prediction threshold for a dataset.

        :param H2OFrame data: the frame with the prediction results for which the confusion matrix should be extracted.
        """
        assert_is_type(data, H2OFrame)
        j = h2o.api("POST /3/Predictions/models/%s/frames/%s" % (self._id, data.frame_id))
        return j["model_metrics"][0]["cm"]["table"]


Instruction:
How would you explain what the following Python 3 function does
def hit_ratio_table(self, train=False, valid=False, xval=False):
        tm = ModelBase._get_metrics(self, train, valid, xval)
        m = {}
        for k, v in zip(list(tm.keys()), list(tm.values())): m[k] = None if v is None else v.hit_ratio_table()
        return list(m.values())[0] if len(m) == 1 else m

Response: Retrieve the Hit Ratios table.


Instruction:
Here you have a function in Python 3, explain what it does
def bytes_iterator(s):
    if s is None: return
    if PY2 or PY3 and isinstance(s, str):
        for ch in s:
            yield ord(ch)
    elif PY3 and isinstance(s, bytes):
        for ch in s:
            yield ch
    else:
        raise TypeError("String argument expected, got %s" % type(s))

Response: Given a string return an iterator over this string s bytes as ints.


Instruction:
Write a Python 3 script to
get object name from first token in line

Response:
def _get_object_name(self, line):
        ''' Get second token in line
        >>> docwriter = ApiDocWriter('sphinx')
        >>> docwriter._get_object_name("  def func():  ")
        'func'
        >>> docwriter._get_object_name("  class Klass(object):  ")
        'Klass'
        >>> docwriter._get_object_name("  class Klass:  ")
        'Klass'
        '''
        name = line.split()[1].split('(')[0].strip()
        # in case we have classes which are not derived from object
        # ie. old style classes
        return name.rstrip(':')


Instruction:
Here you have a function in Python 3, explain what it does
def _uri2path(self, uri):
        ''' Convert uri to absolute filepath

        Parameters
        ----------
        uri : string
            URI of python module to return path for

        Returns
        -------
        path : None or string
            Returns None if there is no valid path for this URI
            Otherwise returns absolute file system path for URI

        Examples
        --------
        >>> docwriter = ApiDocWriter('sphinx')
        >>> import sphinx
        >>> modpath = sphinx.__path__[0]
        >>> res = docwriter._uri2path('sphinx.builder')
        >>> res == os.path.join(modpath, 'builder.py')
        True
        >>> res = docwriter._uri2path('sphinx')
        >>> res == os.path.join(modpath, '__init__.py')
        True
        >>> docwriter._uri2path('sphinx.does_not_exist')

        '''
        if uri == self.package_name:
            return os.path.join(self.root_path, '__init__.py')
        path = uri.replace('.', os.path.sep)
        path = path.replace(self.package_name + os.path.sep, '')
        path = os.path.join(self.root_path, path)
        # XXX maybe check for extensions as well?
        if os.path.exists(path + '.py'): # file
            path += '.py'
        elif os.path.exists(os.path.join(path, '__init__.py')):
            path = os.path.join(path, '__init__.py')
        else:
            return None
        return path

Response: Convert uri to absolute filepath


Instruction:
Create a Python 3 function to
convert directory path to uri

Response:
def _path2uri(self, dirpath):
        ''' Convert directory path to uri '''
        relpath = dirpath.replace(self.root_path, self.package_name)
        if relpath.startswith(os.path.sep):
            relpath = relpath[1:]
        return relpath.replace(os.path.sep, '.')


Instruction:
Write a Python 3 function for
parsing a module defined in uri and return a list of functions and classes.

Response:
def _parse_module(self, uri):
        ''' Parse module defined in *uri* '''
        filename = self._uri2path(uri)
        if filename is None:
            # nothing that we could handle here.
            return ([],[])
        f = open(filename, 'rt')
        functions, classes = self._parse_lines(f)
        f.close()
        return functions, classes


Instruction:
Implement a function in Python 3 to
parse lines of text for functions and classes.

Response:
def _parse_lines(self, linesource):
        ''' Parse lines of text for functions and classes '''
        functions = []
        classes = []
        for line in linesource:
            if line.startswith('def ') and line.count('('):
                # exclude private stuff
                name = self._get_object_name(line)
                if not name.startswith('_'):
                    functions.append(name)
            elif line.startswith('class '):
                # exclude private stuff
                name = self._get_object_name(line)
                if not name.startswith('_'):
                    classes.append(name)
            else:
                pass
        functions.sort()
        classes.sort()
        return functions, classes


Instruction:
Can you implement a function in Python 3 that
generates the API documentation for a module.

Response:
def generate_api_doc(self, uri):
        '''Make autodoc documentation template string for a module

        Parameters
        ----------
        uri : string
            python location of module - e.g 'sphinx.builder'

        Returns
        -------
        S : string
            Contents of API doc
        '''
        # get the names of all classes and functions
        functions, classes = self._parse_module(uri)
        if not len(functions) and not len(classes):
            print 'WARNING: Empty -',uri  # dbg
            return ''

        # Make a shorter version of the uri that omits the package name for
        # titles
        uri_short = re.sub(r'^%s\.' % self.package_name,'',uri)

        ad = '.. AUTO-GENERATED FILE -- DO NOT EDIT!\n\n'

        chap_title = uri_short
        ad += (chap_title+'\n'+ self.rst_section_levels[1] * len(chap_title)
               + '\n\n')

        # Set the chapter title to read 'module' for all modules except for the
        # main packages
        if '.' in uri:
            title = 'Module: :mod:`' + uri_short + '`'
        else:
            title = ':mod:`' + uri_short + '`'
        ad += title + '\n' + self.rst_section_levels[2] * len(title)

        if len(classes):
            ad += '\nInheritance diagram for ``%s``:\n\n' % uri
            ad += '.. inheritance-diagram:: %s \n' % uri
            ad += '   :parts: 3\n'

        ad += '\n.. automodule:: ' + uri + '\n'
        ad += '\n.. currentmodule:: ' + uri + '\n'
        multi_class = len(classes) > 1
        multi_fx = len(functions) > 1
        if multi_class:
            ad += '\n' + 'Classes' + '\n' + \
                  self.rst_section_levels[2] * 7 + '\n'
        elif len(classes) and multi_fx:
            ad += '\n' + 'Class' + '\n' + \
                  self.rst_section_levels[2] * 5 + '\n'
        for c in classes:
            ad += '\n:class:`' + c + '`\n' \
                  + self.rst_section_levels[multi_class + 2 ] * \
                  (len(c)+9) + '\n\n'
            ad += '\n.. autoclass:: ' + c + '\n'
            # must NOT exclude from index to keep cross-refs working
            ad += '  :members:\n' \
                  '  :undoc-members:\n' \
                  '  :show-inheritance:\n' \
                  '  :inherited-members:\n' \
                  '\n' \
                  '  .. automethod:: __init__\n'
        if multi_fx:
            ad += '\n' + 'Functions' + '\n' + \
                  self.rst_section_levels[2] * 9 + '\n\n'
        elif len(functions) and multi_class:
            ad += '\n' + 'Function' + '\n' + \
                  self.rst_section_levels[2] * 8 + '\n\n'
        for f in functions:
            # must NOT exclude from index to keep cross-refs working
            ad += '\n.. autofunction:: ' + uri + '.' + f + '\n\n'
        return ad


Instruction:
Create a Python 3 function to
return True if matchstr does not match patterns .

Response:
def _survives_exclude(self, matchstr, match_type):
        ''' Returns True if *matchstr* does not match patterns

        ``self.package_name`` removed from front of string if present

        Examples
        --------
        >>> dw = ApiDocWriter('sphinx')
        >>> dw._survives_exclude('sphinx.okpkg', 'package')
        True
        >>> dw.package_skip_patterns.append('^\\.badpkg$')
        >>> dw._survives_exclude('sphinx.badpkg', 'package')
        False
        >>> dw._survives_exclude('sphinx.badpkg', 'module')
        True
        >>> dw._survives_exclude('sphinx.badmod', 'module')
        True
        >>> dw.module_skip_patterns.append('^\\.badmod$')
        >>> dw._survives_exclude('sphinx.badmod', 'module')
        False
        '''
        if match_type == 'module':
            patterns = self.module_skip_patterns
        elif match_type == 'package':
            patterns = self.package_skip_patterns
        else:
            raise ValueError('Cannot interpret match type "%s"'
                             % match_type)
        # Match to URI without package name
        L = len(self.package_name)
        if matchstr[:L] == self.package_name:
            matchstr = matchstr[L:]
        for pat in patterns:
            try:
                pat.search
            except AttributeError:
                pat = re.compile(pat)
            if pat.search(matchstr):
                return False
        return True


Instruction:
Write a Python 3 script to
return the list of module names within the current package_name.

Response:
def discover_modules(self):
        ''' Return module sequence discovered from ``self.package_name``

        Parameters
        ----------
        None

        Returns
        -------
        mods : sequence
            Sequence of module names within ``self.package_name``

        Examples
        --------
        >>> dw = ApiDocWriter('sphinx')
        >>> mods = dw.discover_modules()
        >>> 'sphinx.util' in mods
        True
        >>> dw.package_skip_patterns.append('\.util$')
        >>> 'sphinx.util' in dw.discover_modules()
        False
        >>>
        '''
        modules = [self.package_name]
        # raw directory parsing
        for dirpath, dirnames, filenames in os.walk(self.root_path):
            # Check directory names for packages
            root_uri = self._path2uri(os.path.join(self.root_path,
                                                   dirpath))
            for dirname in dirnames[:]: # copy list - we modify inplace
                package_uri = '.'.join((root_uri, dirname))
                if (self._uri2path(package_uri) and
                    self._survives_exclude(package_uri, 'package')):
                    modules.append(package_uri)
                else:
                    dirnames.remove(dirname)
            # Check filenames for modules
            for filename in filenames:
                module_name = filename[:-3]
                module_uri = '.'.join((root_uri, module_name))
                if (self._uri2path(module_uri) and
                    self._survives_exclude(module_uri, 'module')):
                    modules.append(module_uri)
        return sorted(modules)


Instruction:
How would you explain what the following Python 3 function does
def write_api_docs(self, outdir):
        if not os.path.exists(outdir):
            os.mkdir(outdir)
        # compose list of modules
        modules = self.discover_modules()
        self.write_modules_api(modules,outdir)

Response: Generate API reST files for each module in the current directory.


Instruction:
Can you tell what is the following Python 3 function doing
def write_index(self, outdir, froot='gen', relative_to=None):
        if self.written_modules is None:
            raise ValueError('No modules written')
        # Get full filename path
        path = os.path.join(outdir, froot+self.rst_extension)
        # Path written into index is relative to rootpath
        if relative_to is not None:
            relpath = outdir.replace(relative_to + os.path.sep, '')
        else:
            relpath = outdir
        idx = open(path,'wt')
        w = idx.write
        w('.. AUTO-GENERATED FILE -- DO NOT EDIT!\n\n')
        w('.. toctree::\n\n')
        for f in self.written_modules:
            w('   %s\n' % os.path.join(relpath,f))
        idx.close()

Response: Make a reST API index file from written files.


Instruction:
Given the following Python 3 function, write the documentation
def main(argv):
    global g_script_name

    g_script_name = os.path.basename(argv[0])
    parse_config_file()
    parse_args(argv)

    url = 'https://0xdata.atlassian.net/rest/api/2/search?jql=sprint="' + urllib.quote(g_sprint) + '"&maxResults=1000'
    r = requests.get(url, auth=(g_user, g_pass))
    if (r.status_code != 200):
        print("ERROR: status code is " + str(r.status_code))
        sys.exit(1)
    j = r.json()
    issues = j[u'issues']
    pm = PeopleManager()
    for issue in issues:
        pm.add(issue)

    pm.emit()
    print("")

Response:
Main program.

    @return: none


Instruction:
Can you generate the documentation for the following Python 3 function
def to_list(self):
        return [[int(self.table.cell_values[0][1]), int(self.table.cell_values[0][2])],
                [int(self.table.cell_values[1][1]), int(self.table.cell_values[1][2])]]

Response: Convert this confusion matrix into a 2x2 plain list of values.


Instruction:
How would you implement a function in Python 3 that
reads confusion matrices from the list of sources.

Response:
def read_cms(cms=None, domains=None):
        """Read confusion matrices from the list of sources (?)."""
        assert_is_type(cms, [list])
        return [ConfusionMatrix(cm, domains) for cm in cms]


Instruction:
Can you generate the documentation for the following Python 3 function
def load_dict():
    global g_load_java_message_filename
    global g_ok_java_messages

    if os.path.isfile(g_load_java_message_filename):
            # only load dict from file if it exists.
        with open(g_load_java_message_filename,'rb') as ofile:
            g_ok_java_messages = pickle.load(ofile)
    else:   # no previous java messages to be excluded are found
        g_ok_java_messages["general"] = []

Response: Load the java messages that can be ignored pickle file into a dict structure g_ok_java_messages.


Instruction:
Make a summary of the following Python 3 code
def add_new_message():
    global g_new_messages_to_exclude    # filename containing text file from user containing new java ignored messages
    global g_dict_changed               # True if new ignored java messages are added.

    new_message_dict = extract_message_to_dict(g_new_messages_to_exclude)

    if new_message_dict:
        g_dict_changed = True
        update_message_dict(new_message_dict,1)

Response: Add new java messages to ignore from user text file.


Instruction:
Can you tell what is the following Python 3 function doing
def remove_old_message():
    global g_old_messages_to_remove
    global g_dict_changed

    # extract old java ignored messages to be removed in old_message_dict
    old_message_dict = extract_message_to_dict(g_old_messages_to_remove)

    if old_message_dict:
        g_dict_changed = True
        update_message_dict(old_message_dict,2)

Response: Removes old java messages from ignored list if users desired it.


Instruction:
Write a Python 3 function that can
update the g_ok_java_messages dict by the given message_dict.

Response:
def update_message_dict(message_dict,action):
    """
    Update the g_ok_java_messages dict structure by
    1. add the new java ignored messages stored in message_dict if action == 1
    2. remove the java ignored messages stired in message_dict if action == 2.

    Parameters
    ----------

    message_dict :  Python dict
      key: unit test name or "general"
      value: list of java messages that are to be ignored if they are found when running the test stored as the key.  If
        the key is "general", the list of java messages are to be ignored when running all tests.
    action : int
      if 1: add java ignored messages stored in message_dict to g_ok_java_messages dict;
      if 2: remove java ignored messages stored in message_dict from g_ok_java_messages dict.

    :return: none
    """
    global g_ok_java_messages

    allKeys = g_ok_java_messages.keys()

    for key in message_dict.keys():
        if key in allKeys:  # key already exists, just add to it
            for message in message_dict[key]:

                if action == 1:
                    if message not in g_ok_java_messages[key]:
                        g_ok_java_messages[key].append(message)

                if action == 2:
                    if message in g_ok_java_messages[key]:
                        g_ok_java_messages[key].remove(message)
        else:   # new key here.  Can only add and cannot remove
            if action == 1:
                g_ok_java_messages[key] = message_dict[key]


Instruction:
Can you implement a function in Python 3 that
reads in a text file that contains ignored java messages and generates a dictionary structure out of it.

Response:
def extract_message_to_dict(filename):
    """
    Read in a text file that java messages to be ignored and generate a dictionary structure out of
    it with key and value pairs.  The keys are test names and the values are lists of java message
    strings associated with that test name where we are either going to add to the existing java messages
    to ignore or remove them from g_ok_java_messages.

    Parameters
    ----------

    filename :  Str
       filename that contains ignored java messages.  The text file shall contain something like this:
        keyName = general
        Message = nfolds: nfolds cannot be larger than the number of rows (406).
        KeyName = pyunit_cv_cars_gbm.py
        Message = Caught exception: Illegal argument(s) for GBM model: GBM_model_python_1452503348770_2586.  \
            Details: ERRR on field: _nfolds: nfolds must be either 0 or >1.
        ...

    :return:
    message_dict : dict
        contains java message to be ignored with key as unit test name or "general" and values as list of ignored java
        messages.
    """
    message_dict = {}

    if os.path.isfile(filename):
        # open file to read in new exclude messages if it exists
        with open(filename,'r') as wfile:

            key = ""
            val = ""
            startMess = False

            while 1:
                each_line = wfile.readline()

                if not each_line:   # reached EOF
                    if startMess:
                        add_to_dict(val.strip(),key,message_dict)
                    break

                # found a test name or general with values to follow
                if "keyname" in each_line.lower():  # name of test file or the word "general"
                    temp_strings = each_line.strip().split('=')

                    if (len(temp_strings) > 1): # make sure the line is formatted sort of correctly
                        if startMess:   # this is the start of a new key/value pair
                            add_to_dict(val.strip(),key,message_dict)
                            val = ""

                        key = temp_strings[1].strip()
                        startMess = False

                if (len(each_line) > 1) and startMess:
                    val += each_line

                if "ignoredmessage" in each_line.lower():
                    startMess = True    # start of a Java message.
                    temp_mess = each_line.split('=')

                    if (len(temp_mess) > 1):
                        val = temp_mess[1]

    return message_dict


Instruction:
Can you generate a brief explanation for the following Python 3 code
def add_to_dict(val,key,message_dict):
    allKeys = message_dict.keys()
    if (len(val) > 0):    # got a valid message here
        if (key in allKeys) and (val not in message_dict[key]):
            message_dict[key].append(val)   # only include this message if it has not been added before
        else:
            message_dict[key] = [val]

Response: Adds new key val to dict message_dict.


Instruction:
Write a Python 3 script to
save the ignored java message dict stored in g_ok_java_messages into a pickle file for future use.

Response:
def save_dict():
    """
    Save the ignored java message dict stored in g_ok_java_messages into a pickle file for future use.

    :return: none
    """
    global g_ok_java_messages
    global g_save_java_message_filename
    global g_dict_changed

    if g_dict_changed:
        with open(g_save_java_message_filename,'wb') as ofile:
            pickle.dump(g_ok_java_messages,ofile)


Instruction:
Can you write a function in Python 3 where it
writes the java ignored messages in g_ok_java_messages into a text file for humans to read.

Response:
def print_dict():
    """
    Write the java ignored messages in g_ok_java_messages into a text file for humans to read.

    :return: none
    """
    global g_ok_java_messages
    global g_java_messages_to_ignore_text_filename

    allKeys = sorted(g_ok_java_messages.keys())

    with open(g_java_messages_to_ignore_text_filename,'w') as ofile:
        for key in allKeys:

            for mess in g_ok_java_messages[key]:
                ofile.write('KeyName: '+key+'\n')
                ofile.write('IgnoredMessage: '+mess+'\n')

            print('KeyName: ',key)
            print('IgnoredMessage: ',g_ok_java_messages[key])
            print('\n')


Instruction:
Can you tell what is the following Python 3 function doing
def parse_args(argv):
    global g_new_messages_to_exclude
    global g_old_messages_to_remove
    global g_load_java_message_filename
    global g_save_java_message_filename
    global g_print_java_messages

    if len(argv) < 2:   # print out help menu if user did not enter any arguments.
        usage()

    i = 1
    while (i < len(argv)):
        s = argv[i]

        if (s == "--inputfileadd"):         # input text file where new java messages are stored
            i += 1
            if (i > len(argv)):
                usage()
            g_new_messages_to_exclude = argv[i]
        elif (s == "--inputfilerm"):        # input text file containing java messages to be removed from the ignored list
            i += 1
            if (i > len(argv)):
                usage()
            g_old_messages_to_remove = argv[i]
        elif (s == "--loadjavamessage"):    # load previously saved java message pickle file from file other than
            i += 1                          # the default one before performing update
            if i > len(argv):
                usage()
            g_load_java_message_filename = argv[i]
        elif (s == "--savejavamessage"):    # save updated java message in this file instead of default file
            i += 1
            if (i > len(argv)):
                usage()
            g_save_java_message_filename = argv[i]
        elif (s == '--printjavamessage'):   # will print java message out to console and save in a text file
            i += 1
            g_print_java_messages = True
            g_load_java_message_filename = argv[i]
        elif (s == '--help'):               # print help menu and exit
            usage()
        else:
            unknown_arg(s)

        i += 1

Response: Parse user input and set the corresponding global variables to perform the necessary tasks.


Instruction:
Implement a Python 3 function for
illustrating what the various input flags and the options should be.

Response:
def usage():
    """
    Illustrate what the various input flags are and the options should be.

    :return: none
    """
    global g_script_name    # name of the script being run.

    print("")
    print("Usage:  " + g_script_name + " [...options...]")
    print("")
    print("     --help print out this help menu and show all the valid flags and inputs.")
    print("")
    print("    --inputfileadd filename where the new java messages to ignore are stored in.")
    print("")
    print("    --inputfilerm filename where the java messages are removed from the ignored list.")
    print("")
    print("    --loadjavamessage filename pickle file that stores the dict structure containing java messages to include.")
    print("")
    print("    --savejavamessage filename pickle file that saves the final dict structure after update.")
    print("")
    print("    --printjavamessage filename print java ignored java messages stored in pickle file filenam onto console and save into a text file.")
    print("")
    sys.exit(1)


Instruction:
Write a Python 3 function that can
find all python files in the given directory and all subfolders.

Response:
def locate_files(root_dir):
    """Find all python files in the given directory and all subfolders."""
    all_files = []
    root_dir = os.path.abspath(root_dir)
    for dir_name, subdirs, files in os.walk(root_dir):
        for f in files:
            if f.endswith(".py"):
                all_files.append(os.path.join(dir_name, f))
    return all_files


Instruction:
How would you explain what the following Python 3 function does
def find_magic_in_file(filename):
    with open(filename, "rt", encoding="utf-8") as f:
        for line in f:
            if line.startswith("#"):
                comment = line[1:].strip()
                if comment.startswith("~~~~* ") or comment.startswith("----* ") or comment.startswith("====* "):
                    spell = comment[5:].strip()
                    return tuple(spell.split())
            else:
                break
    return None

Response: Search the file for any magic incantations.


Instruction:
Can you generate the documentation for the following Python 3 function
def parse_python_file(filename):

    with open(filename, "rt", encoding="utf-8") as f:
        tokens = list(tokenize.generate_tokens(f.readline))
        tokens = normalize_tokens(tokens)
        module = ChunkCode(tokens, 0, len(tokens))
        module.parse()
        print(module)

Response: Parse a python file into chunks and objects.


Instruction:
Here you have a function in Python 3, explain what it does
def main():
    # magic_files = {}
    for filename in locate_files(ROOT_DIR):
        print("Processing %s" % filename)
        with open(filename, "rt") as f:
            tokens = list(tokenize.generate_tokens(f.readline))
            text1 = tokenize.untokenize(tokens)
            ntokens = normalize_tokens(tokens)
            text2 = tokenize.untokenize(ntokens)
            assert text1 == text2

Response: Executed when script is run as - is.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def init_for_pipeline(self):
        import inspect
        from h2o.transforms.decomposition import H2OPCA
        # check which parameters can be passed to H2OPCA init
        var_names = list(dict(inspect.getmembers(H2OPCA.__init__.__code__))['co_varnames'])
        parameters = {k: v for k, v in self._parms.items() if k in var_names}
        return H2OPCA(**parameters)

Response: Returns H2OPCA object which implements fit and transform method to be used in sklearn. Pipeline properly.


Instruction:
How would you code a function in Python 3 to
transform a H2OFrame using a MOJO Pipeline.

Response:
def transform(self, data, allow_timestamps=False):
        """
        Transform H2OFrame using a MOJO Pipeline.

        :param data: Frame to be transformed.
        :param allow_timestamps: Allows datetime columns to be used directly with MOJO pipelines. It is recommended
        to parse your datetime columns as Strings when using pipelines because pipelines can interpret certain datetime
        formats in a different way. If your H2OFrame is parsed from a binary file format (eg. Parquet) instead of CSV
        it is safe to turn this option on and use datetime columns directly.

        :returns: A new H2OFrame.
        """
        assert_is_type(data, H2OFrame)
        assert_is_type(allow_timestamps, bool)
        return H2OFrame._expr(ExprNode("mojo.pipeline.transform", self.pipeline_id[0], data, allow_timestamps))