Instruction:
Implement a function in Python 3 to
upload given metrics function into H2O cluster.

Response:
def upload_custom_metric(func, func_file="metrics.py", func_name=None, class_name=None, source_provider=None):
    """
    Upload given metrics function into H2O cluster.

    The metrics can have different representation:
      - class: needs to implement map(pred, act, weight, offset, model), reduce(l, r) and metric(l) methods
      - string: the same as in class case, but the class is given as a string

    :param func:  metric representation: string, class
    :param func_file:  internal name of file to save given metrics representation
    :param func_name:  name for h2o key under which the given metric is saved
    :param class_name: name of class wrapping the metrics function (when supplied as string)
    :param source_provider: a function which provides a source code for given function
    :return: reference to uploaded metrics function

    :examples:
        >>> class CustomMaeFunc:
        >>>     def map(self, pred, act, w, o, model):
        >>>         return [abs(act[0] - pred[0]), 1]
        >>>
        >>>     def reduce(self, l, r):
        >>>         return [l[0] + r[0], l[1] + r[1]]
        >>>
        >>>     def metric(self, l):
        >>>         return l[0] / l[1]
        >>>
        >>>
        >>> h2o.upload_custom_metric(CustomMaeFunc, func_name="mae")
        >>>
        >>> custom_func_str = '''class CustomMaeFunc:
        >>>     def map(self, pred, act, w, o, model):
        >>>         return [abs(act[0] - pred[0]), 1]
        >>>
        >>>     def reduce(self, l, r):
        >>>         return [l[0] + r[0], l[1] + r[1]]
        >>>
        >>>     def metric(self, l):
        >>>         return l[0] / l[1]'''
        >>>
        >>>
        >>> h2o.upload_custom_metric(custom_func_str, class_name="CustomMaeFunc", func_name="mae")
    """
    import tempfile
    import inspect

    # Use default source provider
    if not source_provider:
        source_provider = _default_source_provider

    # The template wraps given metrics representation
    _CFUNC_CODE_TEMPLATE = """# Generated code
import water.udf.CMetricFunc as MetricFunc

# User given metric function as a class implementing
# 3 methods defined by interface CMetricFunc
{}

# Generated user metric which satisfies the interface
# of Java MetricFunc
class {}Wrapper({}, MetricFunc, object):
    pass

"""

    assert_satisfies(func, inspect.isclass(func) or isinstance(func, str),
                     "The argument func needs to be string or class !")
    assert_satisfies(func_file, func_file is not None,
                     "The argument func_file is missing!")
    assert_satisfies(func_file, func_file.endswith('.py'),
                     "The argument func_file needs to end with '.py'")
    code = None
    derived_func_name = None
    module_name = func_file[:-3]
    if isinstance(func, str):
        assert_satisfies(class_name, class_name is not None,
                         "The argument class_name is missing! " +
                         "It needs to reference the class in given string!")
        code = _CFUNC_CODE_TEMPLATE.format(func, class_name, class_name)
        derived_func_name = "metrics_{}".format(class_name)
        class_name = "{}.{}Wrapper".format(module_name, class_name)
    else:
        assert_satisfies(func, inspect.isclass(func), "The parameter `func` should be str or class")
        for method in ['map', 'reduce', 'metric']:
            assert_satisfies(func, method in func.__dict__, "The class `func` needs to define method `{}`".format(method))

        assert_satisfies(class_name, class_name is None,
                         "If class is specified then class_name parameter needs to be None")

        class_name = "{}.{}Wrapper".format(module_name, func.__name__)
        derived_func_name = "metrics_{}".format(func.__name__)
        code = _CFUNC_CODE_TEMPLATE.format(source_provider(func), func.__name__, func.__name__)

    # If the func name is not given, use whatever we can derived from given definition
    if not func_name:
        func_name = derived_func_name
    # Saved into jar file
    tmpdir = tempfile.mkdtemp(prefix="h2o-func")
    func_arch_file = _create_zip_file("{}/func.jar".format(tmpdir), (func_file, code))
    # Upload into K/V
    dest_key = _put_key(func_arch_file, dest_key=func_name)
    # Reference
    return "python:{}={}".format(dest_key, class_name)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def main(argv):
    global g_script_name
    global g_tmp_dir

    g_script_name = os.path.basename(argv[0])

    # Override any defaults with the user's choices.
    parse_args(argv)

    # Create tmp dir and clean up on exit with a callback.
    g_tmp_dir = tempfile.mkdtemp(suffix=".tmp_minicran")
    print "Created tmp directory: " + g_tmp_dir
    atexit.register(remove_tmp_dir)

    # Do the work.
    try:
        b = MinicranBuilder(g_print_only, g_output_dir, g_tmp_dir, g_platform, g_rversion, g_branch, g_buildnum)
        b.build()
    except KeyboardInterrupt:
        print("")
        pass

Response: Main function for the nacrator.


Instruction:
How would you code a function in Python 3 to
check that the provided frame id is valid in Rapids language.

Response:
def check_frame_id(frame_id):
    """Check that the provided frame id is valid in Rapids language."""
    if frame_id is None:
        return
    if frame_id.strip() == "":
        raise H2OValueError("Frame id cannot be an empty string: %r" % frame_id)
    for i, ch in enumerate(frame_id):
        # '$' character has special meaning at the beginning of the string; and prohibited anywhere else
        if ch == "$" and i == 0: continue
        if ch not in _id_allowed_characters:
            raise H2OValueError("Character '%s' is illegal in frame id: %s" % (ch, frame_id))
    if re.match(r"-?[0-9]", frame_id):
        raise H2OValueError("Frame id cannot start with a number: %s" % frame_id)


Instruction:
How would you explain what the following Python 3 function does
def _locate(path):

    tmp_dir = os.path.realpath(os.getcwd())
    possible_result = os.path.join(tmp_dir, path)
    while True:
        if os.path.exists(possible_result):
            return possible_result

        next_tmp_dir = os.path.dirname(tmp_dir)
        if next_tmp_dir == tmp_dir:
            raise ValueError("File not found: " + path)

        tmp_dir = next_tmp_dir
        possible_result = os.path.join(tmp_dir, path)

Response: Search for a relative path and turn it into an absolute path.


Instruction:
How would you explain what the following Python 3 function does
def get_human_readable_bytes(size):
    if size == 0: return "0"
    if size is None: return ""
    assert_is_type(size, int)
    assert size >= 0, "`size` cannot be negative, got %d" % size
    suffixes = "PTGMk"
    maxl = len(suffixes)
    for i in range(maxl + 1):
        shift = (maxl - i) * 10
        if size >> shift == 0: continue
        ndigits = 0
        for nd in [3, 2, 1]:
            if size >> (shift + 12 - nd * 3) == 0:
                ndigits = nd
                break
        if ndigits == 0 or size == (size >> shift) << shift:
            rounded_val = str(size >> shift)
        else:
            rounded_val = "%.*f" % (ndigits, size / (1 << shift))
        return "%s %sb" % (rounded_val, suffixes[i] if i < maxl else "")

Response: Convert given number of bytes into a human - readable string.


Instruction:
Can you create a Python 3 function that
converts given duration in milliseconds into a human - readable string.

Response:
def get_human_readable_time(time_ms):
    """
    Convert given duration in milliseconds into a human-readable representation, i.e. hours, minutes, seconds,
    etc. More specifically, the returned string may look like following:
        1 day 3 hours 12 mins
        3 days 0 hours 0 mins
        8 hours 12 mins
        34 mins 02 secs
        13 secs
        541 ms
    In particular, the following rules are applied:
        * milliseconds are printed only if the duration is less than a second;
        * seconds are printed only if the duration is less than an hour;
        * for durations greater than 1 hour we print days, hours and minutes keeping zeros in the middle (i.e. we
          return "4 days 0 hours 12 mins" instead of "4 days 12 mins").

    :param time_ms: duration, as a number of elapsed milliseconds.
    :return: human-readable string representation of the provided duration.
    """
    millis = time_ms % 1000
    secs = (time_ms // 1000) % 60
    mins = (time_ms // 60000) % 60
    hours = (time_ms // 3600000) % 24
    days = (time_ms // 86400000)

    res = ""
    if days > 1:
        res += "%d days" % days
    elif days == 1:
        res += "1 day"

    if hours > 1 or (hours == 0 and res):
        res += " %d hours" % hours
    elif hours == 1:
        res += " 1 hour"

    if mins > 1 or (mins == 0 and res):
        res += " %d mins" % mins
    elif mins == 1:
        res += " 1 min"

    if days == 0 and hours == 0:
        res += " %02d secs" % secs
    if not res:
        res = " %d ms" % millis

    return res.strip()


Instruction:
Can you create a Python 3 function that
prints msg to stdout.

Response:
def print2(msg, flush=False, end="\n"):
    """
    This function exists here ONLY because Sphinx.ext.autodoc gets into a bad state when seeing the print()
    function. When in that state, autodoc doesn't display any errors or warnings, but instead completely
    ignores the "bysource" member-order option.
    """
    print(msg, end=end)
    if flush: sys.stdout.flush()


Instruction:
How would you implement a function in Python 3 that
returns a canonical version of slice s.

Response:
def normalize_slice(s, total):
    """
    Return a "canonical" version of slice ``s``.

    :param slice s: the original slice expression
    :param total int: total number of elements in the collection sliced by ``s``
    :return slice: a slice equivalent to ``s`` but not containing any negative indices or Nones.
    """
    newstart = 0 if s.start is None else max(0, s.start + total) if s.start < 0 else min(s.start, total)
    newstop = total if s.stop is None else max(0, s.stop + total) if s.stop < 0 else min(s.stop, total)
    newstep = 1 if s.step is None else s.step
    return slice(newstart, newstop, newstep)


Instruction:
Write a Python 3 script for
returning True if slice s in normalized form.

Response:
def slice_is_normalized(s):
    """Return True if slice ``s`` in "normalized" form."""
    return (s.start is not None and s.stop is not None and s.step is not None and s.start <= s.stop)


Instruction:
Can you tell what is the following Python 3 function doing
def deprecated(message):
    from traceback import extract_stack
    assert message, "`message` argument in @deprecated is required."

    def deprecated_decorator(fun):
        def decorator_invisible(*args, **kwargs):
            stack = extract_stack()
            assert len(stack) >= 2 and stack[-1][2] == "decorator_invisible", "Got confusing stack... %r" % stack
            print("[WARNING] in %s line %d:" % (stack[-2][0], stack[-2][1]))
            print("    >>> %s" % (stack[-2][3] or "????"))
            print("        ^^^^ %s" % message)
            return fun(*args, **kwargs)

        decorator_invisible.__doc__ = message
        decorator_invisible.__name__ = fun.__name__
        decorator_invisible.__module__ = fun.__module__
        decorator_invisible.__deprecated__ = True
        return decorator_invisible

    return deprecated_decorator

Response: The decorator to mark deprecated functions.


Instruction:
Can you generate the documentation for the following Python 3 function
def join(self):
        self._future = False
        self._job.poll()
        self._job = None

Response: Wait until grid is complete.


Instruction:
How would you explain what the following Python 3 function does
def train(self, x=None, y=None, training_frame=None, offset_column=None, fold_column=None, weights_column=None,
              validation_frame=None, **params):
        """
        Train the model synchronously (i.e. do not return until the model finishes training).

        To train asynchronously call :meth:`start`.

        :param x: A list of column names or indices indicating the predictor columns.
        :param y: An index or a column name indicating the response column.
        :param training_frame: The H2OFrame having the columns indicated by x and y (as well as any
            additional columns specified by fold, offset, and weights).
        :param offset_column: The name or index of the column in training_frame that holds the offsets.
        :param fold_column: The name or index of the column in training_frame that holds the per-row fold
            assignments.
        :param weights_column: The name or index of the column in training_frame that holds the per-row weights.
        :param validation_frame: H2OFrame with validation data to be scored on while training.
        """
        algo_params = locals()
        parms = self._parms.copy()
        parms.update({k: v for k, v in algo_params.items() if k not in ["self", "params", "algo_params", "parms"]})
        # dictionaries have special handling in grid search, avoid the implicit conversion
        parms["search_criteria"] = None if self.search_criteria is None else str(self.search_criteria)
        parms["hyper_parameters"] = None if self.hyper_params  is None else str(self.hyper_params) # unique to grid search
        parms.update({k: v for k, v in list(self.model._parms.items()) if v is not None})  # unique to grid search
        parms.update(params)
        if '__class__' in parms:  # FIXME: hackt for PY3
            del parms['__class__']
        y = algo_params["y"]
        tframe = algo_params["training_frame"]
        if tframe is None: raise ValueError("Missing training_frame")
        if y is not None:
            if is_type(y, list, tuple):
                if len(y) == 1:
                    parms["y"] = y[0]
                else:
                    raise ValueError('y must be a single column reference')
        if x is None:
            if(isinstance(y, int)):
                xset = set(range(training_frame.ncols)) - {y}
            else:
                xset = set(training_frame.names) - {y}
        else:
            xset = set()
            if is_type(x, int, str): x = [x]
            for xi in x:
                if is_type(xi, int):
                    if not (-training_frame.ncols <= xi < training_frame.ncols):
                        raise H2OValueError("Column %d does not exist in the training frame" % xi)
                    xset.add(training_frame.names[xi])
                else:
                    if xi not in training_frame.names:
                        raise H2OValueError("Column %s not in the training frame" % xi)
                    xset.add(xi)
        x = list(xset)
        parms["x"] = x
        self.build_model(parms)

Response: Train the model for the current instance of the class.


Instruction:
Given the following Python 3 function, write the documentation
def predict(self, test_data):
        return {model.model_id: model.predict(test_data) for model in self.models}

Response: Predict on a dataset.


Instruction:
Write a Python 3 function for
returning a dict of models with cross - validated data for the given key.

Response:
def get_xval_models(self, key=None):
        """
        Return a Model object.

        :param str key: If None, return all cross-validated models; otherwise return the model
            specified by the key.
        :returns: A model or a list of models.
        """
        return {model.model_id: model.get_xval_models(key) for model in self.models}


Instruction:
Given the following Python 3 function, write the documentation
def deepfeatures(self, test_data, layer):
        return {model.model_id: model.deepfeatures(test_data, layer) for model in self.models}

Response: Obtain a hidden layer s details on a dataset.


Instruction:
Write a Python 3 function that can
return the frame for the respective weight matrix.

Response:
def weights(self, matrix_id=0):
        """
        Return the frame for the respective weight matrix.

        :param: matrix_id: an integer, ranging from 0 to number of layers, that specifies the weight matrix to return.
        :returns: an H2OFrame which represents the weight matrix identified by matrix_id
        """
        return {model.model_id: model.weights(matrix_id) for model in self.models}


Instruction:
Can you implement a function in Python 3 that
returns the frame for the respective bias vector.

Response:
def biases(self, vector_id=0):
        """
        Return the frame for the respective bias vector.

        :param: vector_id: an integer, ranging from 0 to number of layers, that specifies the bias vector to return.
        :returns: an H2OFrame which represents the bias vector identified by vector_id
        """
        return {model.model_id: model.biases(vector_id) for model in self.models}


Instruction:
Write a Python 3 script to
generate model metrics for this model on test_data.

Response:
def model_performance(self, test_data=None, train=False, valid=False, xval=False):
        """
        Generate model metrics for this model on test_data.

        :param test_data: Data set for which model metrics shall be computed against. All three of train, valid
            and xval arguments are ignored if test_data is not None.
        :param train: Report the training metrics for the model.
        :param valid: Report the validation metrics for the model.
        :param xval: Report the validation metrics for the model.
        :return: An object of class H2OModelMetrics.
        """
        return {model.model_id: model.model_performance(test_data, train, valid, xval) for model in self.models}


Instruction:
Can you tell what is the following Python 3 function doing
def summary(self, header=True):
        table = []
        for model in self.models:
            model_summary = model._model_json["output"]["model_summary"]
            r_values = list(model_summary.cell_values[0])
            r_values[0] = model.model_id
            table.append(r_values)

        # if h2o.can_use_pandas():
        #  import pandas
        #  pandas.options.display.max_rows = 20
        #  print pandas.DataFrame(table,columns=self.col_header)
        #  return
        print()
        if header:
            print('Grid Summary:')
        print()
        H2ODisplay(table, ['Model Id'] + model_summary.col_header[1:], numalign="left", stralign="left")

Response: Print a detailed summary of the explored models.


Instruction:
Write a Python 3 script to
print models sorted by metric.

Response:
def show(self):
        """Print models sorted by metric."""
        hyper_combos = itertools.product(*list(self.hyper_params.values()))
        if not self.models:
            c_values = [[idx + 1, list(val)] for idx, val in enumerate(hyper_combos)]
            print(H2OTwoDimTable(
                col_header=['Model', 'Hyperparameters: [' + ', '.join(list(self.hyper_params.keys())) + ']'],
                table_header='Grid Search of Model ' + self.model.__class__.__name__, cell_values=c_values))
        else:
            print(self.sorted_metric_table())


Instruction:
How would you explain what the following Python 3 function does
def varimp(self, use_pandas=False):
        return {model.model_id: model.varimp(use_pandas) for model in self.models}

Response: Pretty print the variable importances or return them in a list or Pandas DataFrame.


Instruction:
Can you tell what is the following Python 3 function doing
def pprint_coef(self):
        for i, model in enumerate(self.models):
            print('Model', i)
            model.pprint_coef()
            print()

Response: Pretty print the coefficents table.


Instruction:
How would you code a function in Python 3 to
get the hyperparameters of a model explored by grid search.

Response:
def get_hyperparams(self, id, display=True):
        """
        Get the hyperparameters of a model explored by grid search.

        :param str id: The model id of the model with hyperparameters of interest.
        :param bool display: Flag to indicate whether to display the hyperparameter names.

        :returns: A list of the hyperparameters for the specified model.
        """
        idx = id if is_type(id, int) else self.model_ids.index(id)
        model = self[idx]

        # if cross-validation is turned on, parameters in one of the fold model actuall contains the max_runtime_secs
        # parameter and not the main model that is returned.
        if model._is_xvalidated:
            model = h2o.get_model(model._xval_keys[0])

        res = [model.params[h]['actual'][0] if isinstance(model.params[h]['actual'], list)
               else model.params[h]['actual']
               for h in self.hyper_params]
        if display: print('Hyperparameters: [' + ', '.join(list(self.hyper_params.keys())) + ']')
        return res


Instruction:
Write a Python 3 script to
return the dictionary of model parameters derived from the hyper - parameters of interest.

Response:
def get_hyperparams_dict(self, id, display=True):
        """
        Derived and returned the model parameters used to train the particular grid search model.

        :param str id: The model id of the model with hyperparameters of interest.
        :param bool display: Flag to indicate whether to display the hyperparameter names.

        :returns: A dict of model pararmeters derived from the hyper-parameters used to train this particular model.
        """
        idx = id if is_type(id, int) else self.model_ids.index(id)
        model = self[idx]

        model_params = dict()

        # if cross-validation is turned on, parameters in one of the fold model actual contains the max_runtime_secs
        # parameter and not the main model that is returned.
        if model._is_xvalidated:
            model = h2o.get_model(model._xval_keys[0])

        for param_name in self.hyper_names:
            model_params[param_name] = model.params[param_name]['actual'][0] if \
                isinstance(model.params[param_name]['actual'], list) else model.params[param_name]['actual']

        if display: print('Hyperparameters: [' + ', '.join(list(self.hyper_params.keys())) + ']')
        return model_params


Instruction:
Can you generate the documentation for the following Python 3 function
def get_grid(self, sort_by=None, decreasing=None):
        if sort_by is None and decreasing is None: return self

        grid_json = h2o.api("GET /99/Grids/%s" % self._id, data={"sort_by": sort_by, "decreasing": decreasing})
        grid = H2OGridSearch(self.model, self.hyper_params, self._id)
        grid.models = [h2o.get_model(key['name']) for key in grid_json['model_ids']]  # reordered
        first_model_json = h2o.api("GET /99/Models/%s" % grid_json['model_ids'][0]['name'])['models'][0]
        model_class = H2OGridSearch._metrics_class(first_model_json)
        m = model_class()
        m._id = self._id
        m._grid_json = grid_json
        # m._metrics_class = metrics_class
        m._parms = grid._parms
        H2OEstimator.mixin(grid, model_class)
        grid.__dict__.update(m.__dict__.copy())
        return grid

Response: Retrieve an H2OGridSearch instance.


Instruction:
Given the following Python 3 function, write the documentation
def sort_by(self, metric, increasing=True):

        if metric[-1] != ')': metric += '()'
        c_values = [list(x) for x in zip(*sorted(eval('self.' + metric + '.items()'), key=lambda k_v: k_v[1]))]
        c_values.insert(1, [self.get_hyperparams(model_id, display=False) for model_id in c_values[0]])
        if not increasing:
            for col in c_values: col.reverse()
        if metric[-2] == '(': metric = metric[:-2]
        return H2OTwoDimTable(
            col_header=['Model Id', 'Hyperparameters: [' + ', '.join(list(self.hyper_params.keys())) + ']', metric],
            table_header='Grid Search Results for ' + self.model.__class__.__name__,
            cell_values=[list(x) for x in zip(*c_values)])

Response: Returns a table of the hyperparameters sorted by a metric.


Instruction:
Can you tell what is the following Python 3 function doing
def anomaly(self, test_data, per_feature=False):
        return {model.model_id: model.anomaly(test_data, per_feature) for model in self.models}

Response: Calculate the anomaly of the dataset.


Instruction:
Can you generate the documentation for the following Python 3 function
def F1(self, thresholds=None, train=False, valid=False, xval=False):
        return {model.model_id: model.F1(thresholds, train, valid, xval) for model in
                self.models}

Response: Get the F1 values for a set of thresholds for each model in the hierarchy.


Instruction:
Can you write a function in Python 3 where it
gets the confusion matrix for the specified metrics and thresholds.

Response:
def confusion_matrix(self, metrics=None, thresholds=None, train=False, valid=False, xval=False):
        """
        Get the confusion matrix for the specified metrics/thresholds.

        If all are False (default), then return the training metric value.
        If more than one options is set to True, then return a dictionary of metrics where the keys are "train",
        "valid", and "xval".

        :param metrics: A list of metrics or a single metric among ``"min_per_class_accuracy"``, ``"absolute_mcc"``,
            ``"tnr"``, ``"fnr"``, ``"fpr"``, ``"tpr"``, ``"precision"``, ``"accuracy"``, ``"f0point5"``, ``"f2"``,
            ``"f1"``.
        :param thresholds: thresholds parameter must be a list (i.e. [0.01, 0.5, 0.99]). If None, then the thresholds
            in this set of metrics will be used.
        :param bool train: If train is True, then return the confusion matrix value for the training data.
        :param bool valid: If valid is True, then return the confusion matrix value for the validation data.
        :param bool xval:  If xval is True, then return the confusion matrix value for the cross validation data.
        :returns: The confusion matrix for this binomial model.
        """
        return {model.model_id: model.confusion_matrix(metrics, thresholds, train, valid, xval) for model in
                self.models}


Instruction:
Write a Python 3 script for
finding the index in this binomial model s threshold list at which the given threshold is located.

Response:
def find_idx_by_threshold(self, threshold, train=False, valid=False, xval=False):
        """
        Retrieve the index in this metric's threshold list at which the given threshold is located.

        If all are False (default), then return the training metric value.
        If more than one options is set to True, then return a dictionary of metrics where the keys are "train",
        "valid", and "xval".

        :param float threshold: The threshold value to search for.
        :param bool train: If train is True, then return the idx_by_threshold for the training data.
        :param bool valid: If valid is True, then return the idx_by_threshold for the validation data.
        :param bool xval:  If xval is True, then return the idx_by_threshold for the cross validation data.
        :returns: The idx_by_threshold for this binomial model.
        """
        return {model.model_id: model.find_idx_by_threshold(threshold, train, valid, xval) for model in self.models}


Instruction:
Write a Python 3 script for
returning a confusion matrix based of H2O s default prediction threshold for a dataset.

Response:
def confusion_matrix(self, data):
        """
        Returns a confusion matrix based of H2O's default prediction threshold for a dataset.

        :param data: metric for which the confusion matrix will be calculated.
        """
        return {model.model_id: model.confusion_matrix(data) for model in self.models}


Instruction:
Create a Python 3 function for
returning the Importance of components associcated with a pca model.

Response:
def varimp(self, use_pandas=False):
        """
        Return the Importance of components associcated with a pca model.

        use_pandas: ``bool``  (default: ``False``).
        """
        model = self._model_json["output"]
        if "importance" in list(model.keys()) and model["importance"]:
            vals = model["importance"].cell_values
            header = model["importance"].col_header
            if use_pandas and can_use_pandas():
                import pandas
                return pandas.DataFrame(vals, columns=header)
            else:
                return vals
        else:
            print("Warning: This model doesn't have importances of components.")


Instruction:
Given the following Python 3 function, write the documentation
def archetypes(self):
        o = self._model_json["output"]
        yvals = o["archetypes"].cell_values
        archetypes = []
        for yidx, yval in enumerate(yvals):
            archetypes.append(list(yvals[yidx])[1:])
        return archetypes

Response: The archetypes of the GLRM model.


Instruction:
Can you tell what is the following Python 3 function doing
def proj_archetypes(self, test_data, reverse_transform=False):
        if test_data is None or test_data.nrow == 0: raise ValueError("Must specify test data")
        j = h2o.api("POST /3/Predictions/models/%s/frames/%s" % (self.model_id, test_data.frame_id),
                    data={"project_archetypes": True, "reverse_transform": reverse_transform})
        return h2o.get_frame(j["model_metrics"][0]["predictions"]["frame_id"]["name"])

Response: Project archetypes of the model into original feature space.


Instruction:
Can you generate the documentation for the following Python 3 function
def screeplot(self, type="barplot", **kwargs):
        # check for matplotlib. exit if absent.
        is_server = kwargs.pop("server")
        if kwargs:
            raise ValueError("Unknown arguments %s to screeplot()" % ", ".join(kwargs.keys()))
        try:
            import matplotlib
            if is_server: matplotlib.use('Agg', warn=False)
            import matplotlib.pyplot as plt
        except ImportError:
            print("matplotlib is required for this function!")
            return

        variances = [s ** 2 for s in self._model_json['output']['importance'].cell_values[0][1:]]
        plt.xlabel('Components')
        plt.ylabel('Variances')
        plt.title('Scree Plot')
        plt.xticks(list(range(1, len(variances) + 1)))
        if type == "barplot":
            plt.bar(list(range(1, len(variances) + 1)), variances)
        elif type == "lines":
            plt.plot(list(range(1, len(variances) + 1)), variances, 'b--')
        if not is_server: plt.show()

Response: Produce the scree plot.


Instruction:
Explain what the following Python 3 code does
def translate_name(name):
    parts = name.split("_")
    i = 0
    while parts[i] == "":
        parts[i] = "_"
        i += 1
    parts[i] = parts[i].lower()
    for j in range(i + 1, len(parts)):
        parts[j] = parts[j].capitalize()
    i = len(parts) - 1
    while parts[i] == "":
        parts[i] = "_"
        i -= 1
    return "".join(parts)

Response: Translate a name with underscores into camelcase.


Instruction:
Make a summary of the following Python 3 code
def dedent(ind, text):
    text2 = textwrap.dedent(text)
    if ind == 0:
        return text2
    indent_str = " " * ind
    return "\n".join(indent_str + line for line in text2.split("\n"))

Response: dedent text to the specific indentation level.


Instruction:
Write a Python 3 function for
generating the Java class.

Response:
def generate_schema(class_name, schema):
    """
    Generate schema Java class.

    :param class_name: name of the class
    :param schema: information about the class
    """
    superclass = schema["superclass"]
    if superclass == "Schema": superclass = "Object"

    has_map = False
    is_model_builder = False
    has_inherited = False
    for field in schema["fields"]:
        if field["name"] == "__meta": continue
        if field["is_inherited"]:
            has_inherited = True
            continue
        if field["type"].startswith("Map"): has_map = True
        if field["name"] == "can_build": is_model_builder = True

    fields = []
    for field in schema["fields"]:
        if field["name"] == "__meta": continue
        java_type = translate_type(field["type"], field["schema_name"])
        java_value = get_java_value(field)

        # hackery: we flatten the parameters up into the ModelBuilder schema, rather than nesting them in the
        # parameters schema class...
        if False and is_model_builder and field["name"] == "parameters":
            fields.append(("parameters", "null", "ModelParameterSchemaV3[]", field["help"], field["is_inherited"]))
        else:
            fields.append((field["name"], java_value, java_type, field["help"], field["is_inherited"]))

    class_decl = class_name
    if "generics" in schema:
        class_decl += "<" + ", ".join("%s extends %s" % (t, long_type) for t, long_type in schema["generics"]) + ">"
    super_decl = superclass
    if "super_generics" in schema:
        super_decl += "<" + ", ".join(schema["super_generics"]) + ">"

    yield "/*"
    yield " * This file is auto-generated by h2o-3/h2o-bindings/bin/gen_java.py"
    yield " * Copyright 2016 H2O.ai;  Apache License Version 2.0 (see LICENSE for details)"
    yield " */"
    yield "package water.bindings.pojos;"
    yield ""
    yield "import com.google.gson.Gson;"
    yield "import com.google.gson.annotations.*;"
    yield "import java.util.Map;" if has_map else None
    yield ""
    yield ""
    yield "public class %s extends %s {" % (class_decl, super_decl) if super_decl != "Object" else None
    yield "public class %s {" % (class_decl) if super_decl == "Object" else None
    yield ""
    for name, value, ftype, fhelp, inherited in fields:
        if inherited: continue
        ccname = translate_name(name)
        yield "    /**"
        yield bi.wrap(fhelp, indent="     * ")
        yield "     */"
        yield "    @SerializedName(\"%s\")" % name if name != ccname else None
        yield "    public %s %s;" % (ftype, ccname)
        yield ""
    if has_inherited:
        yield ""
        yield "    /*" + ("-" * 114)
        yield "    //" + (" " * 50) + "INHERITED"
        yield "    //" + ("-" * 114)
        yield ""
        for name, value, ftype, fhelp, inherited in fields:
            if not inherited: continue
            yield bi.wrap(fhelp, "    // ")
            yield "    public %s %s;" % (ftype, translate_name(name))
            yield ""
        yield "    */"
        yield ""
    yield "    /**"
    yield "     * Public constructor"
    yield "     */"
    yield "    public %s() {" % class_name
    for name, value, _, _, _ in fields:
        if name == "parameters": continue
        if value == "null": continue
        yield "        %s = %s;" % (translate_name(name), value)
    yield "    }"
    yield ""
    yield "    /**"
    yield "     * Return the contents of this object as a JSON String."
    yield "     */"
    yield "    @Override"
    yield "    public String toString() {"
    yield "        return new Gson().toJson(this);"
    yield "    }"
    yield ""
    yield "}"


Instruction:
How would you explain what the following Python 3 function does
def generate_proxy(classname, endpoints):
    # Replace path vars like (?<schemaname>.*) with {schemaname} for Retrofit's annotation
    var_pattern = re.compile(r"\{(\w+)\}")

    helper_class = []
    found_key_array_parameter = False

    yield "/*"
    yield " * This file is auto-generated by h2o-3/h2o-bindings/bin/gen_java.py"
    yield " * Copyright 2016 H2O.ai;  Apache License Version 2.0 (see LICENSE for details)"
    yield " */"
    yield "package water.bindings.proxies.retrofit;"
    yield ""
    yield "import water.bindings.pojos.*;"
    yield "import retrofit2.*;"
    yield "import retrofit2.http.*;"
    yield "import java.util.Map;" if classname == "Grid" or classname == "ModelBuilders" else None
    yield ""
    yield "public interface " + classname + " {"
    yield ""

    for e in endpoints:
        method = e["handler_method"]
        # should we always use e.api_name ?
        if method == "exec":
            method = e["api_name"]

        param_strs = []
        required_param_strs = []
        for field in e["input_params"]:
            fname = field["name"]
            ftype = "Path" if field["is_path_param"] else "Field"
            ptype = translate_type(field["type"], field["schema_name"])
            if ptype.endswith("KeyV3") or ptype == "ColSpecifierV3": ptype = "String"
            if ptype.endswith("KeyV3[]"): ptype = "String[]"
            param_str = "@{ftype}(\"{fname}\") {ptype} {fname}".format(**locals())
            param_strs.append(param_str)
            if field["required"]:
                required_param_strs.append(param_str)
        if len(param_strs) == len(required_param_strs): required_param_strs = None

        yield u"  /** "
        yield bi.wrap(e["summary"], indent="   * ")
        for field in e["input_params"]:
            s = "   *   @param %s " % field["name"]
            yield s + bi.wrap(field["help"], indent="   *" + " " * (len(s) - 4), indent_first=False)
        yield u"   */"
        # Create 2 versions of each call: first with all input parameters present, and then only with required params
        for params in [param_strs, required_param_strs]:
            if params is None: continue
            yield u"  @FormUrlEncoded" if e["http_method"] == "POST" else None
            yield u"  @{method}(\"{path}\")".format(method=e["http_method"], path=e["url_pattern"])
            if len(params) <= 1:
                args = params[0] if params else ""
                yield "  Call<{schema}> {method}({args});".format(schema=e["output_schema"], method=method, args=args)
            else:
                yield "  Call<{schema}> {method}(".format(schema=e["output_schema"], method=method)
                for arg in params:
                    yield "    " + arg + ("" if arg == params[-1] else ",")
                yield "  );"
            yield ""

        # Make special static Helper class for Grid and ModelBuilders.
        if "algo" in e:
            # We make two train_ and validate_ methods.  One (built here) takes the parameters schema, the other
            # (built above) takes each parameter.
            helper_class.append("    /**")
            helper_class.append(bi.wrap(e["summary"], indent="     * "))
            helper_class.append("     */")
            helper_class.append("    public static Call<{oschema}> {method}({outer_class} z, {ischema} p) {{"
                                .format(ischema=e["input_schema"], oschema=e["output_schema"], method=method,
                                        outer_class=classname))
            helper_class.append("      return z.{method}(".format(method=method))
            for field in e["input_params"]:
                ptype = translate_type(field["type"], field["schema_name"])
                pname = translate_name(field["name"])
                if ptype.endswith("KeyV3"):
                    s = "(p.{parm} == null? null : p.{parm}.name)".format(parm=pname)
                elif ptype.endswith("KeyV3[]"):
                    found_key_array_parameter = True
                    s = "(p.{parm} == null? null : keyArrayToStringArray(p.{parm}))".format(parm=pname)
                elif ptype.startswith("ColSpecifier"):
                    s = "(p.{parm} == null? null : p.{parm}.columnName)".format(parm=pname)
                else:
                    s = "p." + pname
                if field != e["input_params"][-1]:
                    s += ","
                helper_class.append("        " + s)
            helper_class.append("      );")
            helper_class.append("    }")
            helper_class.append("")

    if helper_class:
        yield ""
        yield "  @SuppressWarnings(\"unused\")"
        yield "  class Helper {"
        for line in helper_class:
            yield line
        if found_key_array_parameter:
            yield "    /**"
            yield "     * Return an array of Strings for an array of keys."
            yield "     */"
            yield "    public static String[] keyArrayToStringArray(KeyV3[] keys) {"
            yield "      if (keys == null) return null;"
            yield "      String[] ids = new String[keys.length];"
            yield "      int i = 0;"
            yield "      for (KeyV3 key : keys) ids[i++] = key.name;"
            yield "      return ids;"
            yield "    }"
        yield "  }"
        yield ""

    yield "}"

Response: Generates a list of Retrofit Proxy classes.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def parse(self):
        f = open(self.parse_log_path, "r")
        self.parse2(f)
        f.close()

Response: Parse the log file specified by constructor.


Instruction:
Here you have a function in Python 3, explain what it does
def parse2(self, f):
        line_num = 0
        s = f.readline()
        while (len(s) > 0):
            line_num += 1

            # Check for beginning of parsed data set.
            match_groups = re.search(r"Parse result for (.*) .(\d+) rows.:", s)
            if (match_groups is not None):
                dataset_name = match_groups.group(1)
                if (self.dataset_name is not None):
                    print("ERROR: Too many datasets found on file {} line {}".format(self.parse_log_path, line_num))
                    sys.exit(1)
                self.dataset_name = dataset_name
                num_rows = int(match_groups.group(2))
                self.num_rows = num_rows
                s = f.readline()
                continue

            # Check for numeric column.
            match_groups = re.search(r"INFO WATER:" +
                                     r"\s*C(\d+):" +
                                     r"\s*numeric" +
                                     r"\s*min\((\S*)\)" +
                                     r"\s*max\((\S*).\)" +
                                     r"\s*(na\((\S+)\))?" +
                                     r"\s*(constant)?",
                                     s)
            if (match_groups is not None):
                col_num = int(match_groups.group(1))
                min_val = float(match_groups.group(2))
                max_val = float(match_groups.group(3))
                # four = match_groups.group(4)
                na_count = match_groups.group(5)
                if (na_count is None):
                    na_count = 0
                else:
                    na_count = int(na_count)
                constant_str = match_groups.group(6)
                is_constant = constant_str is not None
                if (is_constant):
                    if (min_val != max_val):
                        print("ERROR: is_constant mismatch on file {} line {}".format(self.parse_log_path, line_num))
                        sys.exit(1)

                na_fraction = float(na_count) / float(self.num_rows)

                is_min_integer = float(int(min_val)) == float(min_val)
                is_max_integer = float(int(min_val)) == float(min_val)
                is_integer = is_min_integer and is_max_integer

                c = RealColumn(col_num, "C" + str(col_num), min_val, max_val, na_fraction, is_integer)
                self.add_col(c)

                s = f.readline()
                continue

            # Check for categorical column.
            match_groups = re.search(r"INFO WATER:" +
                                     r"\s*C(\d+):" +
                                     r"\s*categorical" +
                                     r"\s*min\((\S*)\)" +
                                     r"\s*max\((\S*).\)" +
                                     r"\s*(na\((\S+)\))?" +
                                     r"\s*(constant)?" +
                                     r"\s*cardinality\((\d+)\)",
                                     s)
            if (match_groups is not None):
                col_num = int(match_groups.group(1))
                min_val = float(match_groups.group(2))
                max_val = float(match_groups.group(3))
                # four = match_groups.group(4)
                na_count = match_groups.group(5)
                if (na_count is None):
                    na_count = 0
                else:
                    na_count = int(na_count)
                constant_str = match_groups.group(6)
                is_constant = constant_str is not None
                if (is_constant):
                    if (min_val != max_val):
                        print("ERROR: is_constant mismatch on file {} line {}".format(self.parse_log_path, line_num))
                        sys.exit(1)
                num_levels = int(match_groups.group(7))
                if (is_constant):
                    if (num_levels != 1):
                        print("ERROR: num_levels mismatch on file {} line {}".format(self.parse_log_path, line_num))
                        sys.exit(1)
                na_fraction = float(na_count) / float(self.num_rows)

                c = CategoricalColumn(col_num, "C" + str(col_num), num_levels, na_fraction)
                self.add_col(c)

                s = f.readline()
                continue

            print("ERROR: Unrecognized regexp pattern on file {} line {}".format(self.parse_log_path, line_num))
            sys.exit(1)

Response: Parse the log file specified by constructor.


Instruction:
How would you code a function in Python 3 to
calculate the anomaly of the reconstruction error for the input test_data.

Response:
def anomaly(self, test_data, per_feature=False):
        """
        Obtain the reconstruction error for the input test_data.

        :param H2OFrame test_data: The dataset upon which the reconstruction error is computed.
        :param bool per_feature: Whether to return the square reconstruction error per feature.
            Otherwise, return the mean square error.

        :returns: the reconstruction error.
        """
        if test_data is None or test_data.nrow == 0: raise ValueError("Must specify test data")
        j = h2o.api("POST /3/Predictions/models/%s/frames/%s" % (self.model_id, test_data.frame_id),
                    data={"reconstruction_error": True, "reconstruction_error_per_feature": per_feature})
        return h2o.get_frame(j["model_metrics"][0]["predictions"]["frame_id"]["name"])


Instruction:
Make a summary of the following Python 3 code
def main(argv):
    global g_test_root_dir
    global g_temp_filename

    if len(argv) < 2:
        print("invoke this script as python extractGLRMRuntimeJavaLog.py javatextlog.\n")
        sys.exit(1)
    else:   # we may be in business
        javaLogText = argv[1]         # filename while java log is stored

        print("your java text is {0}".format(javaLogText))
        extractRunInto(javaLogText)

Response: This function is the main function of extractGLRMRuntimeJavaLog. py.


Instruction:
How would you explain what the following Python 3 function does
def _parse_see_also(self, content):
        items = []

        def parse_item_name(text):
            """Match ':role:`name`' or 'name'"""
            m = self._name_rgx.match(text)
            if m:
                g = m.groups()
                if g[1] is None:
                    return g[3], None
                else:
                    return g[2], g[1]
            raise ValueError("%s is not a item name" % text)

        def push_item(name, rest):
            if not name:
                return
            name, role = parse_item_name(name)
            items.append((name, list(rest), role))
            del rest[:]

        current_func = None
        rest = []

        for line in content:
            if not line.strip(): continue

            m = self._name_rgx.match(line)
            if m and line[m.end():].strip().startswith(':'):
                push_item(current_func, rest)
                current_func, line = line[:m.end()], line[m.end():]
                rest = [line.split(':', 1)[1].strip()]
                if not rest[0]:
                    rest = []
            elif not line.startswith(' '):
                push_item(current_func, rest)
                current_func = None
                if ',' in line:
                    for func in line.split(','):
                        push_item(func, [])
                elif line.strip():
                    current_func = line
            elif current_func is not None:
                rest.append(line.strip())
        push_item(current_func, rest)
        return items

Response:
func_name : Descriptive text
            continued text
        another_func_name : Descriptive text
        func_name1, func_name2, :meth:`func_name`, func_name3


Instruction:
Can you tell what is the following Python 3 function doing
def _parse_index(self, section, content):
        def strip_each_in(lst):
            return [s.strip() for s in lst]

        out = {}
        section = section.split('::')
        if len(section) > 1:
            out['default'] = strip_each_in(section[1].split(','))[0]
        for line in content:
            line = line.split(':')
            if len(line) > 2:
                out[line[1]] = strip_each_in(line[2].split(','))
        return out

Response: Parse the index section from the content of a file.


Instruction:
Write a Python 3 function for
filling this instance with values from given dictionary.

Response:
def _fill_from_config(self, config):
        """
        Fill this instance from given dictionary.
        The method only uses keys which corresponds to properties
        this class, throws exception on unknown property name.
        :param conf:  dictionary of parameters
        :return:  a new instance of this class filled with values from given dictionary
        :raises H2OValueError: if input config contains unknown property name.
        """
        for k,v in config.items():
            if k in H2OConnectionConf.allowed_properties:
                setattr(self, k, v)
            else:
                raise H2OValueError(message="Unsupported name of property: %s!" % k, var_name="config")


Instruction:
Create a Python 3 function for
performing a REST API request to the backend H2O server.

Response:
def request(self, endpoint, data=None, json=None, filename=None, save_to=None):
        """
        Perform a REST API request to the backend H2O server.

        :param endpoint: (str) The endpoint's URL, for example "GET /4/schemas/KeyV4"
        :param data: data payload for POST (and sometimes GET) requests. This should be a dictionary of simple
            key/value pairs (values can also be arrays), which will be sent over in x-www-form-encoded format.
        :param json: also data payload, but it will be sent as a JSON body. Cannot be used together with `data`.
        :param filename: file to upload to the server. Cannot be used with `data` or `json`.
        :param save_to: if provided, will write the response to that file (additionally, the response will be
            streamed, so large files can be downloaded seamlessly). This parameter can be either a file name,
            or a folder name. If the folder doesn't exist, it will be created automatically.

        :returns: an H2OResponse object representing the server's response (unless ``save_to`` parameter is
            provided, in which case the output file's name will be returned).
        :raises H2OConnectionError: if the H2O server cannot be reached (or connection is not initialized)
        :raises H2OServerError: if there was a server error (http 500), or server returned malformed JSON
        :raises H2OResponseError: if the server returned an H2OErrorV3 response (e.g. if the parameters were invalid)
        """
        if self._stage == 0: raise H2OConnectionError("Connection not initialized; run .connect() first.")
        if self._stage == -1: raise H2OConnectionError("Connection was closed, and can no longer be used.")

        # Prepare URL
        assert_is_type(endpoint, str)
        match = assert_matches(str(endpoint), r"^(GET|POST|PUT|DELETE|PATCH|HEAD) (/.*)$")
        method = match.group(1)
        urltail = match.group(2)
        url = self._base_url + urltail

        # Prepare data
        if filename is not None:
            assert_is_type(filename, str)
            assert_is_type(json, None, "Argument `json` should be None when `filename` is used.")
            assert_is_type(data, None, "Argument `data` should be None when `filename` is used.")
            assert_satisfies(method, method == "POST",
                             "File uploads can only be done via POST method, got %s" % method)
        elif data is not None:
            assert_is_type(data, dict)
            assert_is_type(json, None, "Argument `json` should be None when `data` is used.")
        elif json is not None:
            assert_is_type(json, dict)

        data = self._prepare_data_payload(data)
        files = self._prepare_file_payload(filename)
        params = None
        if method == "GET" and data:
            params = data
            data = None

        stream = False
        if save_to is not None:
            assert_is_type(save_to, str)
            stream = True

        if self._cookies is not None and isinstance(self._cookies, list):
            self._cookies = ";".join(self._cookies)

        # Make the request
        start_time = time.time()
        try:
            self._log_start_transaction(endpoint, data, json, files, params)

            headers = {"User-Agent": "H2O Python client/" + sys.version.replace("\n", ""),
                       "X-Cluster": self._cluster_id,
                       "Cookie": self._cookies}
            resp = requests.request(method=method, url=url, data=data, json=json, files=files, params=params,
                                    headers=headers, timeout=self._timeout, stream=stream,
                                    auth=self._auth, verify=self._verify_ssl_cert, proxies=self._proxies)
            self._log_end_transaction(start_time, resp)
            return self._process_response(resp, save_to)

        except (requests.exceptions.ConnectionError, requests.exceptions.HTTPError) as e:
            if self._local_server and not self._local_server.is_running():
                self._log_end_exception("Local server has died.")
                raise H2OConnectionError("Local server has died unexpectedly. RIP.")
            else:
                self._log_end_exception(e)
                raise H2OConnectionError("Unexpected HTTP error: %s" % e)
        except requests.exceptions.Timeout as e:
            self._log_end_exception(e)
            elapsed_time = time.time() - start_time
            raise H2OConnectionError("Timeout after %.3fs" % elapsed_time)
        except H2OResponseError as e:
            err = e.args[0]
            err.endpoint = endpoint
            err.payload = (data, json, files, params)
            raise


Instruction:
Create a Python 3 function for
closing an existing connection.

Response:
def close(self):
        """
        Close an existing connection; once closed it cannot be used again.

        Strictly speaking it is not necessary to close all connection that you opened -- we have several mechanisms
        in place that will do so automatically (__del__(), __exit__() and atexit() handlers), however there is also
        no good reason to make this method private.
        """
        if self._session_id:
            try:
                # If the server gone bad, we don't want to wait forever...
                if self._timeout is None: self._timeout = 1
                self.request("DELETE /4/sessions/%s" % self._session_id)
                self._print("H2O session %s closed." % self._session_id)
            except Exception:
                pass
            self._session_id = None
        self._stage = -1


Instruction:
Here you have a function in Python 3, explain what it does
def session_id(self):
        if self._session_id is None:
            req = self.request("POST /4/sessions")
            self._session_id = req.get("session_key") or req.get("session_id")
        return CallableString(self._session_id)

Response: Return the session id of the current connection.


Instruction:
Explain what the following Python 3 code does
def start_logging(self, dest=None):
        assert_is_type(dest, None, str, type(sys.stdout))
        if dest is None:
            dest = os.path.join(tempfile.mkdtemp(), "h2o-connection.log")
        self._print("Now logging all API requests to file %r" % dest)
        self._is_logging = True
        self._logging_dest = dest

Response: Start logging all API requests to a file.


Instruction:
How would you code a function in Python 3 to
prepare the data payload for sending to the server.

Response:
def _prepare_data_payload(data):
        """
        Make a copy of the `data` object, preparing it to be sent to the server.

        The data will be sent via x-www-form-urlencoded or multipart/form-data mechanisms. Both of them work with
        plain lists of key/value pairs, so this method converts the data into such format.
        """
        if not data: return None
        res = {}
        for key, value in viewitems(data):
            if value is None: continue  # don't send args set to None so backend defaults take precedence
            if isinstance(value, list):
                value = stringify_list(value)
            elif isinstance(value, dict):
                if "__meta" in value and value["__meta"]["schema_name"].endswith("KeyV3"):
                    value = value["name"]
                else:
                    value = stringify_dict(value)
            else:
                value = str(value)
            res[key] = value
        return res


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _prepare_file_payload(filename):
        if not filename: return None
        absfilename = os.path.abspath(filename)
        if not os.path.exists(absfilename):
            raise H2OValueError("File %s does not exist" % filename, skip_frames=1)
        return {os.path.basename(absfilename): open(absfilename, "rb")}

Response: Prepare the file payload for sending to the server.


Instruction:
How would you explain what the following Python 3 function does
def _log_start_transaction(self, endpoint, data, json, files, params):
        # TODO: add information about the caller, i.e. which module + line of code called the .request() method
        #       This can be done by fetching current traceback and then traversing it until we find the request function
        self._requests_counter += 1
        if not self._is_logging: return
        msg = "\n---- %d --------------------------------------------------------\n" % self._requests_counter
        msg += "[%s] %s\n" % (time.strftime("%H:%M:%S"), endpoint)
        if params is not None: msg += "     params: {%s}\n" % ", ".join("%s:%s" % item for item in viewitems(params))
        if data is not None:   msg += "     body: {%s}\n" % ", ".join("%s:%s" % item for item in viewitems(data))
        if json is not None:
            import json as j
            msg += "     json: %s\n" % j.dumps(json)
        if files is not None:  msg += "     file: %s\n" % ", ".join(f.name for f in viewvalues(files))
        self._log_message(msg + "\n")

Response: Log the beginning of an API request.


Instruction:
Can you create a Python 3 function that
logs response from an API request.

Response:
def _log_end_transaction(self, start_time, response):
        """Log response from an API request."""
        if not self._is_logging: return
        elapsed_time = int((time.time() - start_time) * 1000)
        msg = "<<< HTTP %d %s   (%d ms)\n" % (response.status_code, response.reason, elapsed_time)
        if "Content-Type" in response.headers:
            msg += "    Content-Type: %s\n" % response.headers["Content-Type"]
        msg += response.text
        self._log_message(msg + "\n\n")


Instruction:
Can you generate the documentation for the following Python 3 function
def _log_message(self, msg):
        if is_type(self._logging_dest, str):
            with open(self._logging_dest, "at", encoding="utf-8") as f:
                f.write(msg)
        else:
            self._logging_dest.write(msg)

Response: Log the message msg to the destination self. _logging_dest.


Instruction:
Write a Python 3 function that can
process a response object and return the full path to the file.

Response:
def _process_response(response, save_to):
        """
        Given a response object, prepare it to be handed over to the external caller.

        Preparation steps include:
           * detect if the response has error status, and convert it to an appropriate exception;
           * detect Content-Type, and based on that either parse the response as JSON or return as plain text.
        """
        status_code = response.status_code
        if status_code == 200 and save_to:
            if save_to.startswith("~"): save_to = os.path.expanduser(save_to)
            if os.path.isdir(save_to) or save_to.endswith(os.path.sep):
                dirname = os.path.abspath(save_to)
                filename = H2OConnection._find_file_name(response)
            else:
                dirname, filename = os.path.split(os.path.abspath(save_to))
            fullname = os.path.join(dirname, filename)
            try:
                if not os.path.exists(dirname):
                    os.makedirs(dirname)
                with open(fullname, "wb") as f:
                    for chunk in response.iter_content(chunk_size=65536):
                        if chunk:  # Empty chunks may occasionally happen
                            f.write(chunk)
            except OSError as e:
                raise H2OValueError("Cannot write to file %s: %s" % (fullname, e))
            return fullname

        content_type = response.headers.get("Content-Type", "")
        if ";" in content_type:  # Remove a ";charset=..." part
            content_type = content_type[:content_type.index(";")]

        # Auto-detect response type by its content-type. Decode JSON, all other responses pass as-is.
        if content_type == "application/json":
            try:
                data = response.json(object_pairs_hook=H2OResponse)
            except (JSONDecodeError, requests.exceptions.ContentDecodingError) as e:
                raise H2OServerError("Malformed JSON from server (%s):\n%s" % (str(e), response.text))
        else:
            data = response.text

        # Success (200 = "Ok", 201 = "Created", 202 = "Accepted", 204 = "No Content")
        if status_code in {200, 201, 202, 204}:
            return data

        # Client errors (400 = "Bad Request", 404 = "Not Found", 412 = "Precondition Failed")
        if status_code in {400, 404, 412} and isinstance(data, (H2OErrorV3, H2OModelBuilderErrorV3)):
            raise H2OResponseError(data)

        # Server errors (notably 500 = "Server Error")
        # Note that it is possible to receive valid H2OErrorV3 object in this case, however it merely means the server
        # did not provide the correct status code.
        raise H2OServerError("HTTP %d %s:\n%r" % (status_code, response.reason, data))


Instruction:
Explain what the following Python 3 code does
def approxEqual(x, y, *args, **kwargs):
    if not (type(x) is type(y) is float):
        # Skip checking for __approxEqual__ in the common case of two floats.
        methodname = '__approxEqual__'
        # Allow the objects to specify what they consider "approximately equal",
        # giving precedence to x. If either object has the appropriate method, we
        # pass on any optional arguments untouched.
        for a,b in ((x, y), (y, x)):
            try:
                method = getattr(a, methodname)
            except AttributeError:
                continue
            else:
                result = method(b, *args, **kwargs)
                if result is NotImplemented:
                    print "WARNING: NotImplemented approxEqual for types"
                    continue
                return bool(result)

    # If we get here without returning, then neither x nor y knows how to do an
    # approximate equal comparison (or are both floats). Fall back to a numeric
    # comparison.
    return _float_approxEqual(x, y, *args, **kwargs)

Response: A simple comparison of two objects x and y.


Instruction:
Explain what the following Python 3 code does
def json_repr(obj, curr_depth=0, max_depth=4):
    def serialize(obj, curr_depth):
        """Recursively walk object's hierarchy. Limit to max_depth"""
        if curr_depth>max_depth:
            return
        if isinstance(obj, (bool, int, long, float, basestring)):
            return obj
        elif isinstance(obj, dict):
            obj = obj.copy()
            for key in obj:
                obj[key] = serialize(obj[key], curr_depth+1)
            return obj
        elif isinstance(obj, list):
            return [serialize(item, curr_depth+1) for item in obj]
        elif isinstance(obj, tuple):
            return tuple(serialize([item for item in obj], curr_depth+1))
        elif hasattr(obj, '__dict__'):
            return serialize(obj.__dict__, curr_depth+1)
        else:
            return repr(obj) # Don't know how to handle, convert to string

    return (serialize(obj, curr_depth+1))

Response: Represent an object as JSON.


Instruction:
Write a Python 3 script for
retrieving information about an AutoML instance.

Response:
def get_automl(project_name):
    """
    Retrieve information about an AutoML instance.

    :param str project_name:  A string indicating the project_name of the automl instance to retrieve.
    :returns: A dictionary containing the project_name, leader model, and leaderboard.
    """
    automl_json = h2o.api("GET /99/AutoML/%s" % project_name)
    project_name = automl_json["project_name"]
    leaderboard_list = [key["name"] for key in automl_json['leaderboard']['models']]

    if leaderboard_list is not None and len(leaderboard_list) > 0:
        leader_id = leaderboard_list[0]
    else:
        leader_id = None

    leader = h2o.get_model(leader_id)
    # Intentionally mask the progress bar here since showing multiple progress bars is confusing to users.
    # If any failure happens, revert back to user's original setting for progress and display the error message.
    is_progress = H2OJob.__PROGRESS_BAR__
    h2o.no_progress()
    try:
        # Parse leaderboard H2OTwoDimTable & return as an H2OFrame
        leaderboard = h2o.H2OFrame(
            automl_json["leaderboard_table"].cell_values,
            column_names=automl_json["leaderboard_table"].col_header)
    except Exception as ex:
        raise ex
    finally:
        if is_progress is True:
            h2o.show_progress()

    leaderboard = leaderboard[1:]
    automl_dict = {'project_name': project_name, "leader": leader, "leaderboard": leaderboard}
    return automl_dict


Instruction:
Make a summary of the following Python 3 code
def train(self, x = None, y = None, training_frame = None, fold_column = None,
              weights_column = None, validation_frame = None, leaderboard_frame = None, blending_frame = None):
        """
        Begins an AutoML task, a background task that automatically builds a number of models
        with various algorithms and tracks their performance in a leaderboard. At any point
        in the process you may use H2O's performance or prediction functions on the resulting
        models.

        :param x: A list of column names or indices indicating the predictor columns.
        :param y: An index or a column name indicating the response column.
        :param fold_column: The name or index of the column in training_frame that holds per-row fold
            assignments.
        :param weights_column: The name or index of the column in training_frame that holds per-row weights.
        :param training_frame: The H2OFrame having the columns indicated by x and y (as well as any
            additional columns specified by fold_column or weights_column).
        :param validation_frame: H2OFrame with validation data. This argument is ignored unless the user sets
            nfolds = 0. If cross-validation is turned off, then a validation frame can be specified and used
            for early stopping of individual models and early stopping of the grid searches.  By default and
            when nfolds > 1, cross-validation metrics will be used for early stopping and thus validation_frame will be ignored.
        :param leaderboard_frame: H2OFrame with test data for scoring the leaderboard.  This is optional and
            if this is set to None (the default), then cross-validation metrics will be used to generate the leaderboard
            rankings instead.
        :param blending_frame: H2OFrame used to train the the metalearning algorithm in Stacked Ensembles (instead of relying on cross-validated predicted values).
            This is optional, but when provided, it is also recommended to disable cross validation
            by setting `nfolds=0` and to provide a leaderboard frame for scoring purposes.

        :returns: An H2OAutoML object.

        :examples:
        >>> # Set up an H2OAutoML object
        >>> aml = H2OAutoML(max_runtime_secs=30)
        >>> # Launch an AutoML run
        >>> aml.train(y=y, training_frame=train)
        """
        ncols = training_frame.ncols
        names = training_frame.names

        #Set project name if None
        if self.project_name is None:
            self.project_name = "automl_" + training_frame.frame_id
            self.build_control["project_name"] = self.project_name

        # Minimal required arguments are training_frame and y (response)
        if y is None:
            raise ValueError('The response column (y) is not set; please set it to the name of the column that you are trying to predict in your data.')
        else:
            assert_is_type(y,int,str)
            if is_type(y, int):
                if not (-ncols <= y < ncols):
                    raise H2OValueError("Column %d does not exist in the training frame" % y)
                y = names[y]
            else:
                if y not in names:
                    raise H2OValueError("Column %s does not exist in the training frame" % y)
            input_spec = {
                'response_column': y,
            }

        if training_frame is None:
            raise ValueError('The training frame is not set!')
        else:
            assert_is_type(training_frame, H2OFrame)
            input_spec['training_frame'] = training_frame.frame_id

        if fold_column is not None:
            assert_is_type(fold_column,int,str)
            input_spec['fold_column'] = fold_column

        if weights_column is not None:
            assert_is_type(weights_column,int,str)
            input_spec['weights_column'] = weights_column

        if validation_frame is not None:
            assert_is_type(validation_frame, H2OFrame)
            input_spec['validation_frame'] = validation_frame.frame_id

        if leaderboard_frame is not None:
            assert_is_type(leaderboard_frame, H2OFrame)
            input_spec['leaderboard_frame'] = leaderboard_frame.frame_id

        if blending_frame is not None:
            assert_is_type(blending_frame, H2OFrame)
            input_spec['blending_frame'] = blending_frame.frame_id

        if self.sort_metric is not None:
            assert_is_type(self.sort_metric, str)
            sort_metric = self.sort_metric.lower()
            # Changed the API to use "deviance" to be consistent with stopping_metric values
            # TO DO: let's change the backend to use "deviance" since we use the term "deviance"
            # After that we can take this `if` statement out
            if sort_metric == "deviance":
                sort_metric = "mean_residual_deviance"
            input_spec['sort_metric'] = sort_metric

        if x is not None:
            assert_is_type(x,list)
            xset = set()
            if is_type(x, int, str): x = [x]
            for xi in x:
                if is_type(xi, int):
                    if not (-ncols <= xi < ncols):
                        raise H2OValueError("Column %d does not exist in the training frame" % xi)
                    xset.add(names[xi])
                else:
                    if xi not in names:
                        raise H2OValueError("Column %s not in the training frame" % xi)
                    xset.add(xi)
            x = list(xset)
            ignored_columns = set(names) - {y} - set(x)
            if fold_column is not None and fold_column in ignored_columns:
                ignored_columns.remove(fold_column)
            if weights_column is not None and weights_column in ignored_columns:
                ignored_columns.remove(weights_column)
            if ignored_columns is not None:
                input_spec['ignored_columns'] = list(ignored_columns)

        automl_build_params = dict(input_spec = input_spec)

        # NOTE: if the user hasn't specified some block of parameters don't send them!
        # This lets the back end use the defaults.
        automl_build_params['build_control'] = self.build_control
        automl_build_params['build_models']  = self.build_models

        resp = h2o.api('POST /99/AutoMLBuilder', json=automl_build_params)
        if 'job' not in resp:
            print("Exception from the back end: ")
            print(resp)
            return

        self._job = H2OJob(resp['job'], "AutoML")
        self._job.poll()
        self._fetch()

Response: This method builds a number of models and runs the metalearning algorithm on the specified data.


Instruction:
Implement a function in Python 3 to
predict on a dataset.

Response:
def predict(self, test_data):
        """
        Predict on a dataset.

        :param H2OFrame test_data: Data on which to make predictions.

        :returns: A new H2OFrame of predictions.

        :examples:
        >>> # Set up an H2OAutoML object
        >>> aml = H2OAutoML(max_runtime_secs=30)
        >>> # Launch an H2OAutoML run
        >>> aml.train(y=y, training_frame=train)
        >>> # Predict with top model from AutoML Leaderboard on a H2OFrame called 'test'
        >>> aml.predict(test)

        """
        if self._fetch():
            self._model = h2o.get_model(self._leader_id)
            return self._model.predict(test_data)
        print("No model built yet...")


Instruction:
Can you generate a brief explanation for the following Python 3 code
def download_pojo(self, path="", get_genmodel_jar=False, genmodel_name=""):

        return h2o.download_pojo(self.leader, path, get_jar=get_genmodel_jar, jar_name=genmodel_name)

Response: Download the POJO for the AutoML leader model in AutoML to the specified path.


Instruction:
Can you implement a function in Python 3 that
downloads the leader model in AutoML in MOJO format.

Response:
def download_mojo(self, path=".", get_genmodel_jar=False, genmodel_name=""):
        """
        Download the leader model in AutoML in MOJO format.

        :param path: the path where MOJO file should be saved.
        :param get_genmodel_jar: if True, then also download h2o-genmodel.jar and store it in folder ``path``.
        :param genmodel_name Custom name of genmodel jar
        :returns: name of the MOJO file written.
        """

        return ModelBase.download_mojo(self.leader, path, get_genmodel_jar, genmodel_name)


Instruction:
How would you explain what the following Python 3 function does
def fit(self, X, y=None, **params):
        if isinstance(self.parms["center"], (tuple, list)): self._means = self.parms["center"]
        if isinstance(self.parms["scale"], (tuple, list)): self._stds = self.parms["scale"]
        if self.means is None and self.parms["center"]:
            self._means = X.mean(return_frame=True).getrow()
        else:
            self._means = False
        if self.stds is None and self.parms["scale"]:
            self._stds = X.sd()
        else:
            self._stds = False
        return self

Response: Fits this object by computing the means and standard deviations used by the transform method.


Instruction:
How would you explain what the following Python 3 function does
def transform(self, X, y=None, **params):
        return X.scale(self.means, self.stds)

Response: Scale an H2OFrame with the fitted means and standard deviations.


Instruction:
Here you have a function in Python 3, explain what it does
def inverse_transform(self, X, y=None, **params):
        for i in range(X.ncol):
            X[i] = self.means[i] + self.stds[i] * X[i]
        return X

Response: Undo the scale transformation.


Instruction:
Can you tell what is the following Python 3 function doing
def main(argv):
    global g_script_name

    g_script_name = os.path.basename(argv[0])
    parse_config_file()
    parse_args(argv)

    url = 'https://0xdata.atlassian.net/rest/api/2/search?jql=' \
          + 'project+in+(PUBDEV,HEXDEV)' \
          + '+and+' \
          + 'resolutiondate+>=+' + g_start_date.strftime("%Y-%m-%d") \
          + '+and+' \
          + 'resolution+in+(Done,Fixed)' \
          + 'order+by+type,component,resolutiondate' \
          + '&maxResults=1000'

    r = requests.get(url, auth=(g_user, g_pass))
    if (r.status_code != 200):
        print("ERROR: status code is " + str(r.status_code))
        sys.exit(1)
    j = r.json()
    issues = j[u'issues']
    if len(issues) >= 1000:
        print("ERROR: len(issues) >= 1000.  Too many issues.")
        sys.exit(1)
    last_issue_type_name = ""
    last_component_name = ""
    for issue in issues:
        issue_type_name = get_issue_type_name(issue)
        component_name = get_issue_component_name(issue)
        if (issue_type_name != last_issue_type_name):
            print("")
            print "# " + issue_type_name
            last_issue_type_name = issue_type_name
        if (component_name != last_component_name):
            print("")
            print "## " + component_name
            last_component_name = component_name
        key = get_issue_key(issue)
        summary = get_issue_summary(issue)
        print "* " + key + ": " + summary

Response:
Main program.

    @return: none


Instruction:
Implement a function in Python 3 to
extract true string from the string_content.

Response:
def extract_true_string(string_content):
    """
    remove extra characters before the actual string we are
    looking for.  The Jenkins console output is encoded using utf-8.  However, the stupid
    redirect function can only encode using ASCII.  I have googled for half a day with no
    results to how to resolve the issue.  Hence, we are going to the heat and just manually
    get rid of the junk.

    Parameters
    ----------

    string_content :  str
        contains a line read in from jenkins console

    :return: str: contains the content of the line after the string '[0m'

    """

    startL,found,endL = string_content.partition('[0m')

    if found:
        return endL
    else:
        return string_content


Instruction:
Can you generate the documentation for the following Python 3 function
def find_time(each_line,temp_func_list):
    global g_weekdays
    global g_months
    global g_failed_test_info_dict

    temp_strings = each_line.strip().split()

    if (len(temp_strings) > 2):
        if ((temp_strings[0] in g_weekdays) or (temp_strings[1] in g_weekdays)) and ((temp_strings[1] in g_months) or (temp_strings[2] in g_months)):
            g_failed_test_info_dict["3.timestamp"] = each_line.strip()
            temp_func_list.remove(find_time)    # found timestamp, don't need to look again for it

    return True

Response: This function finds the approximate date and time from the timestamp of the job we are running on.


Instruction:
Can you implement a function in Python 3 that
finds the slave machine where a Jenkins job was executed on.

Response:
def find_node_name(each_line,temp_func_list):
    """
    Find the slave machine where a Jenkins job was executed on.  It will save this
    information in g_failed_test_info_dict.  In addition, it will
    delete this particular function handle off the temp_func_list as we do not need
    to perform this action again.

    Parameters
    ----------

    each_line :  str
        contains a line read in from jenkins console
    temp_func_list :  list of Python function handles
        contains a list of functions that we want to invoke to extract information from
        the Jenkins console text.

    :return: bool to determine if text mining should continue on the jenkins console text
    """
    global g_node_name
    global g_failed_test_info_dict

    if g_node_name in each_line:
        temp_strings = each_line.split()
        [start,found,endstr] = each_line.partition(g_node_name)

        if found:
            temp_strings = endstr.split()
            g_failed_test_info_dict["6.node_name"] = extract_true_string(temp_strings[1])
            temp_func_list.remove(find_node_name)

    return True


Instruction:
Can you implement a function in Python 3 that
finds the git hash and branch info that a Jenkins job was taken from. It will save this information in g_failed_test_info_dict. In addition, it will delete this particular function handle off the temp_func_list as we do not need to perform this action again. Parameters ---------- each_line : str contains a line read in from jenkins console temp_func_list : list of Python function handles contains a list of functions that we want to invoke to extract information from the Jenkins console text. :return: bool to determine if text mining should continue on the jenkins console text

Response:
def find_git_hash_branch(each_line,temp_func_list):
    """
    Find the git hash and branch info that  a Jenkins job was taken from.  It will save this
    information in g_failed_test_info_dict.  In addition, it will delete this particular
    function handle off the temp_func_list as we do not need to perform this action again.

    Parameters
    ----------

    each_line :  str
        contains a line read in from jenkins console
    temp_func_list :  list of Python function handles
        contains a list of functions that we want to invoke to extract information from
        the Jenkins console text.

    :return: bool to determine if text mining should continue on the jenkins console text
    """
    global g_git_hash_branch
    global g_failed_test_info_dict

    if g_git_hash_branch in each_line:
        [start,found,endstr] = each_line.partition(g_git_hash_branch)
        temp_strings = endstr.strip().split()

        if len(temp_strings) > 1:
            g_failed_test_info_dict["4.git_hash"] = temp_strings[0]
            g_failed_test_info_dict["5.git_branch"] = temp_strings[1]

        temp_func_list.remove(find_git_hash_branch)

    return True


Instruction:
Create a Python 3 function to
find if a Jenkins job has taken too long to finish and was killed.

Response:
def find_build_timeout(each_line,temp_func_list):
    """
    Find if a Jenkins job has taken too long to finish and was killed.  It will save this
    information in g_failed_test_info_dict.

    Parameters
    ----------

    each_line :  str
        contains a line read in from jenkins console
    temp_func_list :  list of Python function handles
        contains a list of functions that we want to invoke to extract information from
        the Jenkins console text.

    :return: bool to determine if text mining should continue on the jenkins console text
"""
    global g_build_timeout
    global g_failed_test_info_dict
    global g_failure_occurred

    if g_build_timeout in each_line:
        g_failed_test_info_dict["8.build_timeout"] = 'Yes'
        g_failure_occurred = True
        return False    # build timeout was found, no need to continue mining the console text
    else:
        return True


Instruction:
Can you write a function in Python 3 where it
finds if a Jenkins job has failed to build.

Response:
def find_build_failure(each_line,temp_func_list):
    """
    Find if a Jenkins job has failed to build.  It will save this
    information in g_failed_test_info_dict.  In addition, it will delete this particular
    function handle off the temp_func_list as we do not need to perform this action again.

    Parameters
    ----------

    each_line :  str
        contains a line read in from jenkins console
    temp_func_list :  list of Python function handles
        contains a list of functions that we want to invoke to extract information from
        the Jenkins console text.

    :return: bool to determine if text mining should continue on the jenkins console text
    """
    global g_build_success
    global g_build_success_tests
    global g_failed_test_info_dict
    global g_failure_occurred
    global g_build_failed_message

    for ind in range(0,len(g_build_failed_message)):
        if g_build_failed_message[ind] in each_line.lower():
            if ((ind == 0) and (len(g_failed_jobs) > 0)):
                continue
            else:
                g_failure_occurred = True
                g_failed_test_info_dict["7.build_failure"] = 'Yes'
                temp_func_list.remove(find_build_failure)
                return False

    return True


Instruction:
Can you tell what is the following Python 3 function doing
def find_java_filename(each_line,temp_func_list):
    global g_before_java_file
    global g_java_filenames

    for each_word in g_before_java_file:
        if (each_word not in each_line):
            return True

    # line contains the name and location of java txt output filename
    temp_strings = each_line.split()
    g_java_filenames.append(temp_strings[-1])

    return True

Response: Find if all the java_0. out. txt files that were mentioned in the Jenkins console output are present in the Jenkins console output.


Instruction:
Can you generate the documentation for the following Python 3 function
def find_build_id(each_line,temp_func_list):
    global g_before_java_file
    global g_java_filenames
    global g_build_id_text
    global g_jenkins_url
    global g_output_filename
    global g_output_pickle_filename

    if g_build_id_text in each_line:
        [startStr,found,endStr] = each_line.partition(g_build_id_text)
        g_failed_test_info_dict["2.build_id"] = endStr.strip()

        temp_func_list.remove(find_build_id)
        g_jenkins_url = os.path.join('http://',g_jenkins_url,'view',g_view_name,'job',g_failed_test_info_dict["1.jobName"],g_failed_test_info_dict["2.build_id"],'artifact')

    return True

Response: This function finds the build id of a jenkins job.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def extract_job_build_url(url_string):
    global g_failed_test_info_dict
    global g_jenkins_url
    global g_view_name

    tempString = url_string.strip('/').split('/')

    if len(tempString) < 6:
        print "Illegal URL resource address.\n"
        sys.exit(1)

    g_failed_test_info_dict["1.jobName"] = tempString[6]

    g_jenkins_url = tempString[2]
    g_view_name = tempString[4]

Response: Extract the job build url from the user input and store it in g_failed_test_info_dict.


Instruction:
Write a Python 3 script for
scanning through the java output text and extract the bad java messages that may or may not happened when unit tests are run. It will not record any bad java messages that are stored in g_ok_java_messages. :return: none

Response:
def grab_java_message():
    """scan through the java output text and extract the bad java messages that may or may not happened when
    unit tests are run.  It will not record any bad java messages that are stored in g_ok_java_messages.

    :return: none
    """

    global g_temp_filename
    global g_current_testname
    global g_java_start_text
    global g_ok_java_messages
    global g_java_general_bad_messages  # store bad java messages not associated with running a unit test
    global g_java_general_bad_message_types
    global g_failure_occurred
    global g_java_message_type
    global g_all_java_message_type
    global g_toContinue

    java_messages = []      # store all bad java messages associated with running a unit test
    java_message_types = [] # store all bad java message types associated with running a unit test

    if os.path.isfile(g_temp_filename): # open temp file containing content of some java_*_0.out.txt
        java_file = open(g_temp_filename,'r')

        g_toContinue = False    # denote if a multi-line message starts

        tempMessage = ""
        messageType = ""

        for each_line in java_file:

            if (g_java_start_text in each_line):
                startStr,found,endStr = each_line.partition(g_java_start_text)

                if len(found) > 0:
                    if len(g_current_testname) > 0: # a new unit test is being started.  Save old info and move on
                        associate_test_with_java(g_current_testname,java_messages,java_message_types)

                    g_current_testname = endStr.strip() # record the test name

                    java_messages = []
                    java_message_types = []

            temp_strings = each_line.strip().split()

            if (len(temp_strings) >= 6) and (temp_strings[5] in g_all_java_message_type):
                if g_toContinue == True:    # at the end of last message fragment
                    addJavaMessages(tempMessage,messageType,java_messages,java_message_types)
                    tempMessage = ""
                    messageType = ""

                # start of new message fragment
                g_toContinue = False
            else: # non standard output.  Continuation of last java message, add it to bad java message list
                if g_toContinue:

                    tempMessage += each_line    # add more java message here
                    # if len(g_current_testname) == 0:
                    #     addJavaMessages(each_line.strip(),"",java_messages,java_message_types)
                    # else:
                    #     addJavaMessages(each_line.strip(),"",java_messages,java_message_types)

            if ((len(temp_strings) > 5) and (temp_strings[5] in g_java_message_type)):  # find a bad java message
                startStr,found,endStr = each_line.partition(temp_strings[5])    # can be WARN,ERRR,FATAL,TRACE

                if found and (len(endStr.strip()) > 0):
                    tempMessage += endStr
                    messageType = temp_strings[5]
#                    if (tempMessage not in g_ok_java_messages["general"]):  # found new bad messages that cannot be ignored
                    g_toContinue = True

                        # add tempMessage to bad java message list
#                        addJavaMessages(tempMessage,temp_strings[5],java_messages,java_message_types)
        java_file.close()


Instruction:
Can you generate a brief explanation for the following Python 3 code
def addJavaMessages(tempMessage,messageType,java_messages,java_message_types):
    global g_current_testname
    global g_java_general_bad_messages
    global g_java_general_bad_message_types
    global g_failure_occurred

    tempMess = tempMessage.strip()

    if (tempMess not in g_ok_java_messages["general"]):
        if (len(g_current_testname) == 0):    # java message not associated with any test name
            g_java_general_bad_messages.append(tempMess)
            g_java_general_bad_message_types.append(messageType)
            g_failure_occurred = True
        else:   # java message found during a test
            write_test = False  # do not include java message for test if False
            if (g_current_testname in g_ok_java_messages.keys()) and (tempMess in g_ok_java_messages[g_current_testname]): # test name associated with ignored Java messages
                write_test = False
            else:   # not java ignored message for current unit test
                write_test = True

            if write_test:
                java_messages.append(tempMess)
                java_message_types.append(messageType)
                g_failure_occurred = True

Response: Adds Java messages to the untagged bad java messages list and returns the empty list if the message is not found in the untagged bad java messages list.


Instruction:
Write a Python 3 function for
looping through java_0. out. txt and extract potentially dangerous WARN or ERRR or FATAL messages associated with a test.

Response:
def extract_java_messages():
    """
    loop through java_*_0.out.txt and extract potentially dangerous WARN/ERRR/FATAL
    messages associated with a test.  The test may even pass but something terrible
    has actually happened.

    :return: none
    """
    global g_jenkins_url
    global g_failed_test_info_dict
    global g_java_filenames

    global g_failed_jobs  # record job names of failed jobs
    global g_failed_job_java_messages # record failed job java message
    global g_failed_job_java_message_types

    global g_success_jobs # record job names of passed jobs
    global g_success_job_java_messages # record of successful jobs bad java messages
    global g_success_job_java_message_types

    global g_java_general_bad_messages  # store java error messages when no job is running
    global g_java_general_bad_message_types # store java error message types when no job is running.

    if (len(g_failed_jobs) > 0):  # artifacts available only during failure of some sort
        for fname in g_java_filenames:  # grab java message from each java_*_*_.out file
            temp_strings = fname.split('/')

            start_url = g_jenkins_url

            for windex in range(6,len(temp_strings)):
                start_url = os.path.join(start_url,temp_strings[windex])
            try:    # first java file path is different.  Can ignore it.
                get_console_out(start_url)  # get java text and save it in local directory for processing
                grab_java_message()         # actually process the java text output and see if we found offensive stuff
            except:
                pass

    # build up the dict structure that we are storing our data in
    if len(g_failed_jobs) > 0:
        g_failed_test_info_dict["failed_tests_info *********"] = [g_failed_jobs,g_failed_job_java_messages,g_failed_job_java_message_types]
    if len(g_success_jobs) > 0:
        g_failed_test_info_dict["passed_tests_info *********"] = [g_success_jobs,g_success_job_java_messages,g_success_job_java_message_types]

    if len(g_java_general_bad_messages) > 0:
        g_failed_test_info_dict["9.general_bad_java_messages"] = [g_java_general_bad_messages,g_java_general_bad_message_types]


Instruction:
Can you write a function in Python 3 where it
saves the log scraping results into the log files denoted by g_output_filename_failed_tests and g_output_filename_passed_tests.

Response:
def save_dict():
    """
    Save the log scraping results into logs denoted by g_output_filename_failed_tests and
    g_output_filename_passed_tests.

    :return: none
    """

    global g_test_root_dir
    global g_output_filename_failed_tests
    global g_output_filename_passed_tests
    global g_output_pickle_filename
    global g_failed_test_info_dict

    # some build can fail really early that no buid id info is stored in the console text.
    if "2.build_id" not in g_failed_test_info_dict.keys():
        g_failed_test_info_dict["2.build_id"] = "unknown"

    build_id = g_failed_test_info_dict["2.build_id"]

    g_output_filename_failed_tests = g_output_filename_failed_tests+'_build_'+build_id+'_failed_tests.log'
    g_output_filename_passed_tests = g_output_filename_passed_tests+'_build_'+build_id+'_passed_tests.log'
    g_output_pickle_filename = g_output_pickle_filename+'_build_'+build_id+'.pickle'

    allKeys = sorted(g_failed_test_info_dict.keys())

    # write out the jenkins job info into log files.
    with open(g_output_pickle_filename,'wb') as test_file:
        pickle.dump(g_failed_test_info_dict,test_file)

    # write out the failure report as text into a text file
    text_file_failed_tests = open(g_output_filename_failed_tests,'w')
    text_file_passed_tests = None
    allKeys = sorted(g_failed_test_info_dict.keys())
    write_passed_tests = False

    if ("passed_tests_info *********" in allKeys):
        text_file_passed_tests = open(g_output_filename_passed_tests,'w')
        write_passed_tests = True

    for keyName in allKeys:
        val = g_failed_test_info_dict[keyName]
        if isinstance(val,list):    # writing one of the job lists
            if (len(val) == 3):     # it is a message for a test
                if keyName == "failed_tests_info *********":
                    write_test_java_message(keyName,val,text_file_failed_tests)

                if keyName == "passed_tests_info *********":
                    write_test_java_message(keyName,val,text_file_passed_tests)
            elif (len(val) == 2):                   # it is a general bad java message
                write_java_message(keyName,val,text_file_failed_tests)
                if write_passed_tests:
                    write_java_message(keyName,val,text_file_passed_tests)
        else:
            write_general_build_message(keyName,val,text_file_failed_tests)
            if write_passed_tests:
                write_general_build_message(keyName,val,text_file_passed_tests)

    text_file_failed_tests.close()
    if write_passed_tests:
        text_file_passed_tests.close()


Instruction:
Write a Python 3 function that can
write key value into log file when value is a string and not a list.

Response:
def write_general_build_message(key,val,text_file):
    """
    Write key/value into log file when the value is a string and not a list.

    Parameters
    ----------
    key :  str
        key value in g_failed_test_info_dict
    value :  str
        corresponding value associated with the key in key
    text_file : file handle
        file handle of log file to write the info to.

    :return: none
    """
    text_file.write(key+": ")
    text_file.write(val)
    text_file.write('\n\n')


Instruction:
Create a Python 3 function for
writing the content of the log file into the summary text file.

Response:
def write_file_content(fhandle,file2read):
    """
    Write one log file into the summary text file.

    Parameters
    ----------
    fhandle :  Python file handle
        file handle to the summary text file
    file2read : Python file handle
        file handle to log file where we want to add its content to the summary text file.

    :return: none
    """
    if os.path.isfile(file2read):

        # write summary of failed tests logs
        with open(file2read,'r') as tfile:
            fhandle.write('============ Content of '+ file2read)
            fhandle.write('\n')
            fhandle.write(tfile.read())
            fhandle.write('\n\n')