Instruction:
Given the following Python 3 function, write the documentation
def calculate_ideal_atomic_percent(pif):
    if not isinstance(pif, ChemicalSystem):
        return pif
    if not pif.chemical_formula:
        return pif
    else:
        expanded_formula_no_special_char = _expand_formula_(
            pif.chemical_formula)
        element_array = _create_emprical_compositional_array_(
            expanded_formula_no_special_char)
        appended_e_array = _add_atomic_percents_(element_array)
        for e in appended_e_array:
            # Checks if a Composition element decribing that element already
            # exists.
            if _get_element_in_pif_composition_(pif, e["symbol"]):
                # If it exists, it removes the old Composition object, and
                # inserts a new one with ideal atomic percent added.
                in_pif = _get_element_in_pif_composition_(pif, e["symbol"])
                comp = in_pif[0]
                pif.composition.pop(in_pif[1])
                comp.idealAtomicPercent = e["atomic_percent"]
                pif.composition.append(comp)
            else:
                # If not, it creates a new Composition object with the element
                # and ideal atomic percent.
                comp = Composition()
                comp.element = e["symbol"]
                comp.idealAtomicPercent = e["atomic_percent"]
                pif.composition.append(comp)
        return pif

Response: Calculates ideal atomic percents from a chemical formula string from a pif. Returns an appended pif with composition elements modified or added.


Instruction:
Write a Python 3 script to
calculate ideal atomic weight percents from a chemical formula string from a pif. Returns an appended pif with composition elements modified or added.

Response:
def calculate_ideal_weight_percent(pif):
    """
    Calculates ideal atomic weight percents from a chemical formula string from a pif. Returns an appended pif with composition elements modified or added.

    :param pif: a ChemicalSystem pif
    :return: modified pif object
    """
    if not isinstance(pif, ChemicalSystem):
        return pif
    if not pif.chemical_formula:
        return pif
    else:
        expanded_formula_no_special_char = _expand_formula_(
            pif.chemical_formula)
        element_array = _create_emprical_compositional_array_(
            expanded_formula_no_special_char)
        appended_e_array = _add_ideal_atomic_weights_(element_array)
        a_array_with_pcts = _add_ideal_weight_percent_(appended_e_array)
        for e in a_array_with_pcts:
            # Checks if a Composition element decribing that element already
            # exists.
            if _get_element_in_pif_composition_(pif, e["symbol"]):
                # If it exists, it removes the old Composition object, and
                # inserts a new one with ideal atomic weight percent added
                in_pif = _get_element_in_pif_composition_(pif, e["symbol"])
                comp = in_pif[0]
                pif.composition.pop(in_pif[1])
                comp.idealWeightPercent = e["weight_percent"]
                pif.composition.append(comp)
            else:
                # If not, it creates a new Composition object with the element
                # and ideal atomic weight percent.
                comp = Composition()
                comp.element = e["symbol"]
                comp.idealWeightPercent = e["weight_percent"]
                pif.composition.append(comp)
        return pif


Instruction:
Create a Python 3 function to
expand a chemical formula string into a non - emperical but expanded form.

Response:
def _expand_formula_(formula_string):
    """
    Accounts for the many ways a user may write a formula string, and returns an expanded chemical formula string.
    Assumptions:
    -The Chemical Formula string it is supplied is well-written, and has no hanging parethneses
    -The number of repeats occurs after the elemental symbol or ) ] character EXCEPT in the case of a hydrate where it is assumed to be in front of the first element
    -All hydrates explicitly use the · symbol
    -Only (, (,[, ], ., · are "important" symbols to intrepreting the string.
    -IONS ARE NOT HANDLED

    :param formula_string: a messy chemical formula string
    :return: a non-emperical but expanded formula string
    """
    formula_string = re.sub(r'[^A-Za-z0-9\(\)\[\]\·\.]+', '', formula_string)
    hydrate_pos = formula_string.find('·')
    if hydrate_pos >= 0:
        formula_string = _expand_hydrate_(hydrate_pos, formula_string)
    search_result = re.search(
        r'(?:[\(\[]([A-Za-z0-9]+)[\)\]](\d*))',
        formula_string)
    if search_result is None:
        return formula_string
    this_start = search_result.start()
    this_end = search_result.end()
    this_string = search_result.group()
    this_expansion_array = re.findall(
        r'(?:[\(\[]([A-Za-z0-9]+)[\)\]](\d*))', this_string)
    for a in this_expansion_array:
        if a[1] == "":
            a = (a[0], 1)
        parenth_expanded = ""
        multiplier = float(a[1])
        element_array = re.findall('[A-Z][^A-Z]*', a[0])
        for e in element_array:
            occurance_array = re.findall('[0-9][^0-9]*', e)
            if len(occurance_array) == 0:
                occurance_array.append(1)
            for o in occurance_array:
                symbol = re.findall('[A-Z][a-z]*', e)
                total_num = float(o) * multiplier
                if total_num.is_integer():
                    total_num = int(total_num)
                total_str = str(total_num)
                if total_str == "1":
                    total_str = ""
                new_string = symbol[0] + total_str
                parenth_expanded += new_string
        formula_string = formula_string[0:this_start] + \
            parenth_expanded + formula_string[this_end:]
        return _expand_formula_(formula_string)


Instruction:
Make a summary of the following Python 3 code
def _expand_hydrate_(hydrate_pos, formula_string):
    hydrate = formula_string[hydrate_pos + 1:]
    hydrate_string = ""
    multiplier = float(re.search(r'^[\d\.]+', hydrate).group())
    element_array = re.findall('[A-Z][^A-Z]*', hydrate)
    for e in element_array:
        occurance_array = re.findall('[0-9][^0-9]*', e)
        if len(occurance_array) == 0:
            occurance_array.append(1)
        for o in occurance_array:
            symbol = re.findall('[A-Z][a-z]*', e)
            total_num = float(o) * multiplier
            if total_num.is_integer():
                total_num = int(total_num)
            total_str = str(total_num)
            if total_str == "1":
                total_str = ""
            new_string = symbol[0] + total_str
            hydrate_string += new_string
    return formula_string[:hydrate_pos] + hydrate_string

Response: Expands the hydrate portion of a chemical formula string to all elements of the chemical formula.


Instruction:
Here you have a function in Python 3, explain what it does
def _create_compositional_array_(expanded_chemical_formaula_string):
    element_array = re.findall(
        '[A-Z][^A-Z]*',
        expanded_chemical_formaula_string)
    split_element_array = []
    for s in element_array:
        m = re.match(r"([a-zA-Z]+)([0-9\.]*)", s, re.I)
        if m:
            items = m.groups()
            if items[1] == "":
                items = (items[0], 1)
        this_e = {"symbol": items[0], "occurances": float(items[1])}

        split_element_array.append(this_e)
    return split_element_array

Response: Splits an expanded chemical formula string into an array of dictionaries containing information about each element.


Instruction:
Write a Python 3 script to
consolidate an elemental array of element dictionaries with no repeating elements.

Response:
def _consolidate_elemental_array_(elemental_array):
    """
    Accounts for non-empirical chemical formulas by taking in the compositional array generated by _create_compositional_array_() and returning a consolidated array of dictionaries with no repeating elements

    :param elemental_array: an elemental array generated from _create_compositional_array_()
    :return: an array of element dictionaries
    """
    condensed_array = []
    for e in elemental_array:
        exists = False
        for k in condensed_array:
            if k["symbol"] == e["symbol"]:
                exists = True
                k["occurances"] += e["occurances"]
                break
        if not exists:
            condensed_array.append(e)
    return condensed_array


Instruction:
Can you tell what is the following Python 3 function doing
def _add_ideal_atomic_weights_(elemental_array):
    for a in elemental_array:
        this_atomic_weight = elements_data[a["symbol"]]["atomic_weight"]
        a["weight"] = a["occurances"] * this_atomic_weight
    return elemental_array

Response: Adds the atomic weight property to each of the dictionaries in elemental_array


Instruction:
Write a Python 3 function that can
add the weight_percent property to each of the dictionaries in elemental_array

Response:
def _add_ideal_weight_percent_(elemental_array):
    """
    Adds the "weight_percent" property to each of the dictionaries in elemental_array

    :param elemental_array: an array of dictionaries containing information about the elements in the system
    :return: the appended elemental_array
    """
    t_mass = _calculate_total_mass_(elemental_array)
    for a in elemental_array:
        a["weight_percent"] = a["weight"] / t_mass * 100
    return elemental_array


Instruction:
Can you tell what is the following Python 3 function doing
def _add_atomic_percents_(elemental_array):
    n_atoms = _calculate_n_atoms_(elemental_array)
    for e in elemental_array:
        e["atomic_percent"] = e["occurances"] / n_atoms * 100
    return elemental_array

Response: Adds ideal atomic percents to the emperical compositional element array generated using _create_emprical_compositional_array_


Instruction:
Implement a function in Python 3 to
return the Composition object and the position in the composition array if the element in question if in the composition array is equal to the atomic symbol of the element in question if in the composition array otherwise returns False

Response:
def _get_element_in_pif_composition_(pif, elemental_symbol):
    """
    If the element in question if in the composition array in the pif, it returns that Composition object and the position in the composition array otherwise it returns False

    :param pif: ChemicalSystem Pif in question
    :param elemental_symbol: string of the atomic symbol of the element in question
    :return: either False if not found in the composition or the Compositional object along with its index in the composition array in the pif
    """
    if pif.composition is None:
        pif.composition = []
    for i, c in enumerate(pif.composition):
        if c.element == elemental_symbol or c.element.lower(
        ) == elements_data[elemental_symbol]["name"].lower():
            return [c, i]
        i += 1
    return False


Instruction:
How would you explain what the following Python 3 function does
def parse_name_string(full_name):
    name = Name()
    if "," in full_name:
        toks = full_name.split(",")
        name.family = toks[0]
        name.given = ",".join(toks[1:]).strip()
    else:
        toks = full_name.split()
        name.given = toks[0]
        name.family = " ".join(toks[1:]).strip()
    return name

Response: Parse a full name into a Name object


Instruction:
How would you explain what the following Python 3 function does
def creator_to_person(creator):
    name = Name()
    if "creatorName" in creator:
        name = parse_name_string(creator["creatorName"])
    if "familyName" in creator:
        name.family = creator["familyName"]
    if "givenName" in creator:
        name.given = creator["givenName"]

    person = Person(name=name, tags=creator.get("affiliations"))
    return person

Response: Parse the creator block in datacite into a Person object


Instruction:
Can you create a Python 3 function that
parses a top - level datacite dictionary into a Reference object

Response:
def datacite_to_pif_reference(dc):
    """
    Parse a top-level datacite dictionary into a Reference
    :param dc: dictionary containing datacite metadata
    :return: Reference corresponding to that datacite entry
    """
    ref = Reference()
    if dc.get('identifier', {}).get('identifierType') == "DOI":
        ref.doi = dc.get('identifier', {}).get('identifier')
    ref.title = dc.get('title')
    ref.publisher = dc.get('publisher')
    ref.year = dc.get('publicationYear')

    ref.authors = [creator_to_person(x).name for x in dc.get('creators', [])] or None

    return ref


Instruction:
Write a Python 3 script to
evaluate a query and return a list of MDF records

Response:
def query_to_mdf_records(query=None, dataset_id=None, mdf_acl=None):
    """Evaluate a query and return a list of MDF records

    If a datasetID is specified by there is no query, a simple
    whole dataset query is formed for the user
    """
    if not query and not dataset_id:
        raise ValueError("Either query or dataset_id must be specified")
    if query and dataset_id:
        raise ValueError("Both query and dataset_id were specified; pick one or the other.")
    if not query:
        query = PifSystemReturningQuery(
            query=DataQuery(
                dataset=DatasetQuery(
                    id=Filter(equal=dataset_id)
                )
            ),
            size = 10000 # Don't pull down all the results by default
        )

    client = get_client()

    if not mdf_acl:
        raise ValueError('Access controls (mdf_acl) must be specified.  Use ["public"] for public access')

    pif_result = client.pif_search(query)
    if len(pif_result.hits) == 0:
        return []

    example_uid = pif_result.hits[0].system.uid
    dataset_query = DatasetReturningQuery(
        query=DataQuery(
            system=PifSystemQuery(
                uid=Filter(equal=example_uid)
            )
        ),
        size = 1 # we only expect one dataset to hit
    )

    dataset_result = client.dataset_search(dataset_query)

    records = []
    for hit in pif_result.hits:
        records.append(pif_to_mdf_record(hit.system, dataset_result.hits[0], mdf_acl))

    return records


Instruction:
Here you have a function in Python 3, explain what it does
def pif_to_mdf_record(pif_obj, dataset_hit, mdf_acl):
    res = {}
    res["mdf"] = _to_meta_data(pif_obj, dataset_hit, mdf_acl)
    res[res["mdf"]["source_name"]] = _to_user_defined(pif_obj)
    return dumps(res)

Response: Convert a PIF into a MDF record


Instruction:
Can you tell what is the following Python 3 function doing
def _to_meta_data(pif_obj, dataset_hit, mdf_acl):
    pif = pif_obj.as_dictionary()
    dataset = dataset_hit.as_dictionary()
    mdf = {}
    try:
        if pif.get("names"):
            mdf["title"] = pif["names"][0]
        else:
            mdf["title"] = "Citrine PIF " + str(pif["uid"])

        if pif.get("chemicalFormula"):
            mdf["composition"] = pif["chemicalFormula"]
        elif pif.get("composition"):
            mdf["composition"] = ''.join([comp["element"] for comp in pif["composition"] if comp["element"]])
        if not mdf["composition"]:
            mdf.pop("composition")

        mdf["acl"] = mdf_acl
        mdf["source_name"] = _construct_new_key(dataset["name"])

        if pif.get("contacts"):
            mdf["data_contact"] = []
            for contact in pif["contacts"]:
                data_c = {
                    "given_name": contact["name"]["given"],  #REQ
                    "family_name": contact["name"]["family"]  #REQ
                    }
                if contact.get("email"):
                    data_c["email"] = contact.get("email", "")
                if contact.get("orcid"):
                    data_c["orcid"] = contact.get("orcid", "")
                mdf["data_contact"].append(data_c)
            if not mdf["data_contact"]:
                mdf.pop("data_contact")

        mdf["data_contributor"] = [{}]
        if "owner" in dataset:
            name = dataset["owner"].split()
            contributor = {
                "given_name": name[0],
                "family_name": name[1],
                "email": dataset["email"]
            }
            mdf["data_contributor"] = [contributor]

        mdf["links"] = {
            "landing_page": "https://citrination.com/datasets/{}".format(dataset["id"]),
            "publication": []
            }
        if pif.get("references"):
            mdf["author"] = []
            mdf["citation"] = []
            for ref in pif["references"]:
                if ref.get("doi"):
                    mdf["citation"].append(ref["doi"]) #TODO: Make actual citation
                    mdf["links"]["publication"].append(ref["doi"])
                if ref.get("authors"):
                    for author in ref["authors"]:
                        if author.get("given") and author.get("family"):
                            mdf["author"].append({
                                "given_name": author["given"],
                                "family_name": author["family"]
                                })
            # Remove fields if blank
            if not mdf["author"]:
                mdf.pop("author")
            if not mdf["citation"]:
                mdf.pop("citation")
        if not mdf["links"]["publication"]:
            mdf["links"].pop("publication")

        if pif.get("licenses", [{}])[0].get("url"):
            mdf["license"] = pif["licenses"][0]["url"]
        if pif.get("tags"):
            mdf["tags"] = pif["tags"]

    # If required MDF metadata is missing from PIF, abort
    except KeyError as e:
        print("Error: Required MDF metadata", str(e), "not found in PIF", pif["uid"])
        return None

    return mdf

Response: Convert the meta - data from the PIF into the MDF.


Instruction:
Write a Python 3 function that can
read the system in the PIF to populate the user - defined portion of the system.

Response:
def _to_user_defined(pif_obj):
    """Read the systems in the PIF to populate the user-defined portion"""
    res = {}

    # make a read view to flatten the hierarchy
    rv = ReadView(pif_obj)

    # Iterate over the keys in the read view
    for k in rv.keys():
        name, value = _extract_key_value(rv[k].raw)
        # add any objects that can be extracted
        if name and value is not None:
            res[name] = value

    # Grab interesting values not in the ReadView
    pif = pif_obj.as_dictionary()

    elements = {}
    if pif.get("composition"):
        for comp in pif["composition"]:
            if comp.get("actualAtomicPercent"):
                elements[comp["element"]] = float(comp["actualAtomicPercent"]["value"])
            elif comp.get("actualWeightPercent"):
                elements[comp["element"]] = float(comp["actualWeightPercent"]["value"])
        if elements:
            res["elemental_percent"] = elements
    elif pif.get("chemicalFormula"):
        symbol = ""
        num = ""
        # Chemical formulae are comprised of letters, numbers, and potentially characters we don't care about
        for char in pif["chemicalFormula"]:
            # Uppercase char indicates beginning of new symbol
            if char.isupper():
                # If there is already a symbol in holding, process it
                if symbol:
                    try:
                        elements[symbol] = int(num)
                    # If num is a float, raises ValueError
                    except ValueError:
                        elements[symbol] = float(num) if num else 1
                    symbol = ""
                    num = ""
                symbol += char
            # Lowercase chars or digits are continuations of a symbol
            elif char.islower():
                symbol += char
            elif char.isdigit():
                num += char
            elif char == ".":
                num += char
            # All other chars are not useful
        if elements:
            res["elemental_proportion"] = elements
    return res


Instruction:
Write a Python 3 script for
constructing a MDF safe key from the name and units

Response:
def _construct_new_key(name, units=None):
    """Construct an MDF safe key from the name and units"""
    to_replace = ["/", "\\", "*", "^", "#", " ", "\n", "\t", ",", ".", ")", "(", "'", "`", "-"]
    to_remove = ["$", "{", "}"]

    cat = name
    if units:
        cat = "_".join([name, units])
    for c in to_replace:
       cat = cat.replace(c, "_")
    for c in to_remove:
       cat = cat.replace(c, "")

    cat = re.sub('_+','_', cat)

    return cat


Instruction:
Here you have a function in Python 3, explain what it does
def _extract_key_value(obj):
    key = None; value = None

    # Parse a Value object, which includes Properties
    if isinstance(obj, Value):
        key = _construct_new_key(obj.name, obj.units)
        value = []
        if obj.scalars:
            value = [(val.value if isinstance(val, Scalar) else val)
                     for val in obj.scalars]
        elif obj.vectors and len(obj.vectors) == 1:
            value = [(val.value if isinstance(val, Scalar) else val)
                     for val in obj.vectors[0]]
        if len(value) == 1:
            value = value[0]
        elif len(value) == 0:
            value = None

    # If there is a process step, pul out its name as the value
    # TODO: resolve duplicates
    if isinstance(obj, ProcessStep):
        key = "Processing"
        value = obj.name

    return key, value

Response: Extract the value from the object and make a descriptive key


Instruction:
How would you explain what the following Python 3 function does
def process_input_buffer():
    from polysh.control_commands_helpers import handle_control_command
    data = the_stdin_thread.input_buffer.get()
    remote_dispatcher.log(b'> ' + data)

    if data.startswith(b':'):
        try:
            handle_control_command(data[1:-1].decode())
        except UnicodeDecodeError as e:
            console_output(b'Could not decode command.')
        return

    if data.startswith(b'!'):
        try:
            retcode = subprocess.call(data[1:], shell=True)
        except OSError as e:
            if e.errno == errno.EINTR:
                console_output(b'Child was interrupted\n')
                retcode = 0
            else:
                raise
        if retcode > 128 and retcode <= 192:
            retcode = 128 - retcode
        if retcode > 0:
            console_output('Child returned {:d}\n'.format(retcode).encode())
        elif retcode < 0:
            console_output('Child was terminated by signal {:d}\n'.format(
                -retcode).encode())
        return

    for r in dispatchers.all_instances():
        try:
            r.dispatch_command(data)
        except asyncore.ExitNow as e:
            raise e
        except Exception as msg:
            raise msg
            console_output('{} for {}, disconnecting\n'.format(
                str(msg), r.display_name).encode())
            r.disconnect()
        else:
            if r.enabled and r.state is remote_dispatcher.STATE_IDLE:
                r.change_state(remote_dispatcher.STATE_RUNNING)

Response: Send the content of the input buffer to all remote processes


Instruction:
Make a summary of the following Python 3 code
def write_main_socket(c):
    the_stdin_thread.socket_write.send(c)
    while True:
        try:
            the_stdin_thread.socket_write.recv(1)
        except socket.error as e:
            if e.errno != errno.EINTR:
                raise
        else:
            break

Response: Synchronous write to the main socket wait for ACK


Instruction:
Write a Python 3 function that can
try to get the PID of the stdin thread otherwise get the whole process ID

Response:
def get_stdin_pid(cached_result=None):
    """Try to get the PID of the stdin thread, otherwise get the whole process
    ID"""
    if cached_result is None:
        try:
            tasks = os.listdir('/proc/self/task')
        except OSError as e:
            if e.errno != errno.ENOENT:
                raise
            cached_result = os.getpid()
        else:
            tasks.remove(str(os.getpid()))
            assert len(tasks) == 1
            cached_result = int(tasks[0])
    return cached_result


Instruction:
Can you generate the documentation for the following Python 3 function
def interrupt_stdin_thread():
    dupped_stdin = os.dup(0)  # Backup the stdin fd
    assert not the_stdin_thread.interrupt_asked  # Sanity check
    the_stdin_thread.interrupt_asked = True  # Not user triggered
    os.lseek(tempfile_fd, 0, 0)  # Rewind in the temp file
    os.dup2(tempfile_fd, 0)  # This will make raw_input() return
    pid = get_stdin_pid()
    os.kill(pid, signal.SIGWINCH)  # Try harder to wake up raw_input()
    the_stdin_thread.out_of_raw_input.wait()  # Wait for this return
    the_stdin_thread.interrupt_asked = False  # Restore sanity
    os.dup2(dupped_stdin, 0)  # Restore stdin
    os.close(dupped_stdin)

Response: Interrupt the stdin thread.


Instruction:
Here you have a function in Python 3, explain what it does
def add(self, data):
        assert isinstance(data, bytes)
        with self.lock:
            self.buf += data

Response: Add data to the buffer


Instruction:
Given the following Python 3 function, write the documentation
def get(self):
        data = b''
        with self.lock:
            data, self.buf = self.buf, b''

        return data

Response: Get the content of the buffer


Instruction:
Write a Python 3 script to
handle all the available character commands in the socket.

Response:
def handle_read(self):
        """Handle all the available character commands in the socket"""
        while True:
            try:
                c = self.recv(1)
            except socket.error as e:
                if e.errno == errno.EWOULDBLOCK:
                    return
                else:
                    raise
            else:
                self._do(c)
                self.socket.setblocking(True)
                self.send(b'A')
                self.socket.setblocking(False)


Instruction:
Make a summary of the following Python 3 code
def decker_sum(a, b):
    x = a + b
    y = b - (x - a) if abs(a) > abs(b) else a - (x - b)
    return x, y

Response: Compute the knuth sum of two numbers.


Instruction:
Make a summary of the following Python 3 code
def ksum(p, K=2):
    # Don't override the input data.
    q = p.copy()
    distill(q, K - 1)
    return numpy.sum(q[:-1], axis=0) + q[-1]

Response: Sum the data in a sequence of K elements.


Instruction:
Can you write a function in Python 3 where it
enables or disable the specified shells.

Response:
def toggle_shells(command, enable):
    """Enable or disable the specified shells. If the command would have
    no effect, it changes all other shells to the inverse enable value."""
    selection = list(selected_shells(command))
    if command and command != '*' and selection:
        for i in selection:
            if i.state != remote_dispatcher.STATE_DEAD and i.enabled != enable:
                break
        else:
            toggle_shells('*', not enable)

    for i in selection:
        if i.state != remote_dispatcher.STATE_DEAD:
            i.set_enabled(enable)


Instruction:
Make a summary of the following Python 3 code
def selected_shells(command):
    if not command or command == '*':
        for i in dispatchers.all_instances():
            yield i
        return
    selected = set()
    instance_found = False
    for pattern in command.split():
        found = False
        for expanded_pattern in expand_syntax(pattern):
            for i in dispatchers.all_instances():
                instance_found = True
                if fnmatch(i.display_name, expanded_pattern):
                    found = True
                    if i not in selected:
                        selected.add(i)
                        yield i
        if instance_found and not found:
            console_output('{} not found\n'.format(pattern).encode())

Response: Iterator over the shells with names matching the patterns.


Instruction:
Can you tell what is the following Python 3 function doing
def complete_shells(line, text, predicate=lambda i: True):
    res = [i.display_name + ' ' for i in dispatchers.all_instances() if
           i.display_name.startswith(text) and
           predicate(i) and
           ' ' + i.display_name + ' ' not in line]
    return res

Response: Return the shell names to include in the completion


Instruction:
Explain what the following Python 3 code does
def kdot(x, y, K=2):
    xx = x.reshape(-1, x.shape[-1])
    yy = y.reshape(y.shape[0], -1)

    xx = numpy.ascontiguousarray(xx)
    yy = numpy.ascontiguousarray(yy)

    r = _accupy.kdot_helper(xx, yy).reshape((-1,) + x.shape[:-1] + y.shape[1:])
    return ksum(r, K - 1)

Response: Algorithm 5. 10. Dot product algorithm in K - fold working precision


Instruction:
Explain what the following Python 3 code does
def kill_all():
    for i in dispatchers.all_instances():
        try:
            os.kill(-i.pid, signal.SIGKILL)
        except OSError:
            # The process was already dead, no problem
            pass

Response: Kill all the remote shells


Instruction:
Can you generate the documentation for the following Python 3 function
def main():

    sentry_dsn = os.environ.get('POLYSH_SENTRY_DSN')

    if sentry_dsn:
        from raven import Client
        client = Client(
            dsn=sentry_dsn,
            release='.'.join(map(str, VERSION)),
            ignore_exceptions=[
                KeyboardInterrupt
            ]
        )

        try:
            run()
        except Exception:
            client.captureException()

    else:
        run()

Response: Wrapper around run to setup sentry


Instruction:
Write a Python 3 function for
handling a read chunk of data from the socket.

Response:
def _handle_read_chunk(self):
        """Some data can be read"""
        new_data = b''
        buffer_length = len(self.read_buffer)
        try:
            while buffer_length < self.MAX_BUFFER_SIZE:
                try:
                    piece = self.recv(4096)
                except OSError as e:
                    if e.errno == errno.EAGAIN:
                        # End of the available data
                        break
                    elif e.errno == errno.EIO and new_data:
                        # Hopefully we could read an error message before the
                        # actual termination
                        break
                    else:
                        raise

                if not piece:
                    # A closed connection is indicated by signaling a read
                    # condition, and having recv() return 0.
                    break

                new_data += piece
                buffer_length += len(piece)

        finally:
            new_data = new_data.replace(b'\r', b'\n')
            self.read_buffer += new_data
        return new_data


Instruction:
Write a Python 3 script for
augmenting the buffer with stuff to write when possible

Response:
def dispatch_write(self, buf):
        """Augment the buffer with stuff to write when possible"""
        self.write_buffer += buf
        if len(self.write_buffer) > self.MAX_BUFFER_SIZE:
            console_output('Buffer too big ({:d}) for {}\n'.format(
                len(self.write_buffer), str(self)).encode())
            raise asyncore.ExitNow(1)
        return True


Instruction:
Can you generate the documentation for the following Python 3 function
def safe_write(buf):
    assert isinstance(buf, bytes)
    while True:
        try:
            os.write(1, buf)
            break
        except IOError as e:
            if e.errno != errno.EINTR:
                raise

Response: Write a buffer to the internal memory.


Instruction:
Write a Python 3 script to
use instead of print to clear the status information before printing

Response:
def console_output(msg, logging_msg=None):
    """Use instead of print, to clear the status information before printing"""
    assert isinstance(msg, bytes)
    assert isinstance(logging_msg, bytes) or logging_msg is None

    from polysh import remote_dispatcher

    remote_dispatcher.log(logging_msg or msg)
    if remote_dispatcher.options.interactive:
        from polysh.stdin import the_stdin_thread
        the_stdin_thread.no_raw_input()
        global last_status_length
        if last_status_length:
            safe_write('\r{}\r'.format(
                last_status_length * ' ').encode())
            last_status_length = 0
    safe_write(msg)


Instruction:
Can you create a Python 3 function that
returns a tuple with the number of awaited processes and the total number of all active resources.

Response:
def count_awaited_processes():
    """Return a tuple with the number of awaited processes and the total
    number"""
    awaited = 0
    total = 0
    for i in all_instances():
        if i.enabled:
            total += 1
            if i.state is not remote_dispatcher.STATE_IDLE:
                awaited += 1
    return awaited, total


Instruction:
Can you generate the documentation for the following Python 3 function
def update_terminal_size():
    w, h = terminal_size()
    w = max(w - display_names.max_display_name_length - 2, min(w, 10))
    # python bug http://python.org/sf/1112949 on amd64
    # from ajaxterm.py
    bug = struct.unpack('i', struct.pack('I', termios.TIOCSWINSZ))[0]
    packed_size = struct.pack('HHHH', h, w, 0, 0)
    term_size = w, h
    for i in all_instances():
        if i.enabled and i.term_size != term_size:
            i.term_size = term_size
            fcntl.ioctl(i.fd, bug, packed_size)

Response: Propagate the terminal size to the remote shells accounting for the longest name


Instruction:
Write a Python 3 script to
turn a 2 - dimension list of bytes into a 1 - dimension list of bytes with correct spacing

Response:
def format_info(info_list):
    """Turn a 2-dimension list of bytes into a 1-dimension list of bytes with
    correct spacing"""

    max_lengths = []
    if info_list:
        nr_columns = len(info_list[0])
    else:
        nr_columns = 0
    for i in range(nr_columns):
        max_lengths.append(max([len(info[i]) for info in info_list]))

    flattened_info_list = []
    for info_id in range(len(info_list)):
        info = info_list[info_id]
        for str_id in range(len(info) - 1):
            # Don't justify the last column (i.e. the last printed line)
            # as it can get much longer in some shells than in others
            orig_str = info[str_id]
            indent = max_lengths[str_id] - len(orig_str)
            info[str_id] = orig_str + indent * b' '
        flattened_info_list.append(b' '.join(info) + b'\n')

    return flattened_info_list


Instruction:
Can you tell what is the following Python 3 function doing
def generate_ill_conditioned_dot_product(n, c, dps=100):
    # Algorithm 6.1 from
    #
    # ACCURATE SUM AND DOT PRODUCT,
    # TAKESHI OGITA, SIEGFRIED M. RUMP, AND SHIN'ICHI OISHI.
    assert n >= 6
    n2 = round(n / 2)
    x = numpy.zeros(n)
    y = numpy.zeros(n)

    b = math.log2(c)
    # vector of exponents between 0 and b/2:
    e = numpy.rint(numpy.random.rand(n2) * b / 2).astype(int)
    # make sure exponents b/2 and 0 actually occur in e
    # vectors x,y
    e[0] = round(b / 2) + 1
    e[-1] = 0

    # generate first half of vectors x, y
    rx, ry = numpy.random.rand(2, n2)
    x[:n2] = (2 * rx - 1) * 2 ** e
    y[:n2] = (2 * ry - 1) * 2 ** e

    def dot_exact(x, y):
        mp.dps = dps
        # convert to list first, see
        # <https://github.com/fredrik-johansson/mpmath/pull/385>
        return mp.fdot(x.tolist(), y.tolist())

    # for i=n2+1:n and v=1:i,
    #     generate x_i, y_i such that (*) x(v)’*y(v) ~ 2^e(i-n2)
    # generate exponents for second half
    e = numpy.rint(numpy.linspace(b / 2, 0, n - n2)).astype(int)
    rx, ry = numpy.random.rand(2, n2)
    for i in range(n2, n):
        # x_i random with generated exponent
        x[i] = (2 * rx[i - n2] - 1) * 2 ** e[i - n2]
        # y_i according to (*)
        y[i] = (
            (2 * ry[i - n2] - 1) * 2 ** e[i - n2] - dot_exact(x[: i + 1], y[: i + 1])
        ) / x[i]

    x, y = numpy.random.permutation((x, y))
    # the true dot product rounded to nearest floating point
    d = dot_exact(x, y)
    # the actual condition number
    C = 2 * dot_exact(abs(x), abs(y)) / abs(d)

    return x, y, d, C

Response: Generate a random illumination conditioned DOT product.


Instruction:
Write a Python 3 function for
returning the number of RemoteDispatcher.handle_read() calls made by this iteration

Response:
def main_loop_iteration(timeout=None):
    """Return the number of RemoteDispatcher.handle_read() calls made by this
    iteration"""
    prev_nr_read = nr_handle_read
    asyncore.loop(count=1, timeout=timeout, use_poll=True)
    return nr_handle_read - prev_nr_read


Instruction:
Write a Python 3 script to
launch the ssh command in the child process

Response:
def launch_ssh(self, name, port):
        """Launch the ssh command in the child process"""
        if options.user:
            name = '%s@%s' % (options.user, name)
        evaluated = options.ssh % {'host': name, 'port': port}
        if evaluated == options.ssh:
            evaluated = '%s %s' % (evaluated, name)
        os.execlp('/bin/sh', 'sh', '-c', evaluated)


Instruction:
Explain what the following Python 3 code does
def change_state(self, state):
        if state is not self.state:
            if self.debug:
                self.print_debug(b'state => ' + STATE_NAMES[state].encode())
            if self.state is STATE_NOT_STARTED:
                self.read_in_state_not_started = b''
            self.state = state

Response: Change the state of the remote process logging the change


Instruction:
How would you explain what the following Python 3 function does
def disconnect(self):
        try:
            os.kill(-self.pid, signal.SIGKILL)
        except OSError:
            # The process was already dead, no problem
            pass
        self.read_buffer = b''
        self.write_buffer = b''
        self.set_enabled(False)
        if self.read_in_state_not_started:
            self.print_lines(self.read_in_state_not_started)
            self.read_in_state_not_started = b''
        if options.abort_error and self.state is STATE_NOT_STARTED:
            raise asyncore.ExitNow(1)
        self.change_state(STATE_DEAD)

Response: Disconnects from the remote process.


Instruction:
Implement a function in Python 3 to
configure the terminal to use the termios module.

Response:
def configure_tty(self):
        """We don't want \n to be replaced with \r\n, and we disable the echo"""
        attr = termios.tcgetattr(self.fd)
        attr[1] &= ~termios.ONLCR  # oflag
        attr[3] &= ~termios.ECHO  # lflag
        termios.tcsetattr(self.fd, termios.TCSANOW, attr)
        # unsetopt zle prevents Zsh from resetting the tty
        return b'unsetopt zle 2> /dev/null;stty -echo -onlcr -ctlecho;'


Instruction:
Can you tell what is the following Python 3 function doing
def set_prompt(self):
        # No right prompt
        command_line = b'PS2=;RPS1=;RPROMPT=;'
        command_line += b'PROMPT_COMMAND=;'
        command_line += b'TERM=ansi;'
        command_line += b'unset HISTFILE;'
        prompt1, prompt2 = callbacks.add(b'prompt', self.seen_prompt_cb, True)
        command_line += b'PS1="' + prompt1 + b'""' + prompt2 + b'\n"\n'
        return command_line

Response: Set the prompt for the process holding the lock.


Instruction:
Create a Python 3 function for
handling the fast case of reading the entries in the log file.

Response:
def handle_read_fast_case(self, data):
        """If we are in a fast case we'll avoid the long processing of each
        line"""
        if self.state is not STATE_RUNNING or callbacks.any_in(data):
            # Slow case :-(
            return False

        last_nl = data.rfind(b'\n')
        if last_nl == -1:
            # No '\n' in data => slow case
            return False
        self.read_buffer = data[last_nl + 1:]
        self.print_lines(data[:last_nl])
        return True


Instruction:
Write a Python 3 script to
handle a single line of the read buffer.

Response:
def handle_read(self):
        """We got some output from a remote shell, this is one of the state
        machine"""
        if self.state == STATE_DEAD:
            return
        global nr_handle_read
        nr_handle_read += 1
        new_data = self._handle_read_chunk()
        if self.debug:
            self.print_debug(b'==> ' + new_data)
        if self.handle_read_fast_case(self.read_buffer):
            return
        lf_pos = new_data.find(b'\n')
        if lf_pos >= 0:
            # Optimization: we knew there were no '\n' in the previous read
            # buffer, so we searched only in the new_data and we offset the
            # found index by the length of the previous buffer
            lf_pos += len(self.read_buffer) - len(new_data)
        elif self.state is STATE_NOT_STARTED and \
                options.password is not None and \
                b'password:' in self.read_buffer.lower():
            self.dispatch_write('{}\n'.format(options.password).encode())
            self.read_buffer = b''
            return
        while lf_pos >= 0:
            # For each line in the buffer
            line = self.read_buffer[:lf_pos + 1]
            if callbacks.process(line):
                pass
            elif self.state in (STATE_IDLE, STATE_RUNNING):
                self.print_lines(line)
            elif self.state is STATE_NOT_STARTED:
                self.read_in_state_not_started += line
                if b'The authenticity of host' in line:
                    msg = line.strip(b'\n') + b' Closing connection.'
                    self.disconnect()
                elif b'REMOTE HOST IDENTIFICATION HAS CHANGED' in line:
                    msg = b'Remote host identification has changed.'
                else:
                    msg = None

                if msg:
                    self.print_lines(msg + b' Consider manually connecting or '
                                     b'using ssh-keyscan.')

            # Go to the next line in the buffer
            self.read_buffer = self.read_buffer[lf_pos + 1:]
            if self.handle_read_fast_case(self.read_buffer):
                return
            lf_pos = self.read_buffer.find(b'\n')
        if self.state is STATE_NOT_STARTED and not self.init_string_sent:
            self.dispatch_write(self.init_string)
            self.init_string_sent = True


Instruction:
Can you generate a brief explanation for the following Python 3 code
def print_unfinished_line(self):
        if self.state is STATE_RUNNING:
            if not callbacks.process(self.read_buffer):
                self.print_lines(self.read_buffer)
            self.read_buffer = b''

Response: Print the unfinished line of the log.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def handle_write(self):
        num_sent = self.send(self.write_buffer)
        if self.debug:
            if self.state is not STATE_NOT_STARTED or options.password is None:
                self.print_debug(b'<== ' + self.write_buffer[:num_sent])
        self.write_buffer = self.write_buffer[num_sent:]

Response: Let s write as much as we can.


Instruction:
Can you implement a function in Python 3 that
logs some debugging information to the console

Response:
def print_debug(self, msg):
        """Log some debugging information to the console"""
        assert isinstance(msg, bytes)
        state = STATE_NAMES[self.state].encode()
        console_output(b'[dbg] ' + self.display_name.encode() + b'[' + state +
                       b']: ' + msg + b'\n')


Instruction:
Given the following Python 3 function, write the documentation
def get_info(self):
        return [self.display_name.encode(),
                self.enabled and b'enabled' or b'disabled',
                STATE_NAMES[self.state].encode() + b':',
                self.last_printed_line.strip()]

Response: Return a list with all information available about this process


Instruction:
Can you implement a function in Python 3 that
dispatches a buffer to the appropriate handler.

Response:
def dispatch_write(self, buf):
        """There is new stuff to write when possible"""
        if self.state != STATE_DEAD and self.enabled:
            super().dispatch_write(buf)
            return True
        return False


Instruction:
Write a Python 3 function that can
change the name of the shell possibly updating the maximum name length

Response:
def change_name(self, new_name):
        """Change the name of the shell, possibly updating the maximum name
        length"""
        if not new_name:
            name = self.hostname
        else:
            name = new_name.decode()
        self.display_name = display_names.change(
            self.display_name, name)


Instruction:
Create a Python 3 function for
sending to the remote shell its new name to be shell expanded

Response:
def rename(self, name):
        """Send to the remote shell, its new name to be shell expanded"""
        if name:
            # defug callback add?
            rename1, rename2 = callbacks.add(
                b'rename', self.change_name, False)
            self.dispatch_command(b'/bin/echo "' + rename1 + b'""' + rename2 +
                                  b'"' + name + b'\n')
        else:
            self.change_name(self.hostname.encode())


Instruction:
Can you generate a brief explanation for the following Python 3 code
def terminal_size():  # decide on *some* terminal size
    cr = _ioctl_GWINSZ(0) or _ioctl_GWINSZ(
        1) or _ioctl_GWINSZ(2)  # try open fds
    if not cr:                                                  # ...then ctty
        try:
            fd = os.open(os.ctermid(), os.O_RDONLY)
            cr = _ioctl_GWINSZ(fd)
            os.close(fd)
        except BaseException:
            pass
        if not cr:                            # env vars or finally defaults
            try:
                cr = os.environ['LINES'], os.environ['COLUMNS']
            except BaseException:
                cr = 25, 80
    return int(cr[1]), int(cr[0])

Response: Return the number of lines and columns of a terminal.


Instruction:
Make a summary of the following Python 3 code
def complete(text, state):
    global completion_results
    if state == 0:
        line = readline.get_line_buffer()
        if line.startswith(':'):
            # Control command completion
            completion_results = complete_control_command(line, text)
        else:
            if line.startswith('!') and text and line.startswith(text):
                dropped_exclam = True
                text = text[1:]
            else:
                dropped_exclam = False
            completion_results = []
            # Complete local paths
            completion_results += complete_local_path(text)
            # Complete from history
            l = len(text)
            completion_results += [w + ' ' for w in history_words if
                                   len(w) > l and w.startswith(text)]
            if readline.get_begidx() == 0:
                # Completing first word from $PATH
                completion_results += [w + ' ' for w in user_commands_in_path
                                           if len(w) > l and w.startswith(text)]
            completion_results = remove_dupes(completion_results)
            if dropped_exclam:
                completion_results = ['!' + r for r in completion_results]

    if state < len(completion_results):
        return completion_results[state]
    completion_results = None
    return None

Response: Completes the given text with the next possible completion.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def Popen(*args, **kwargs):

    read_line = None
    if 'read_line' in kwargs:
        read_line = kwargs['read_line']
        del kwargs['read_line']

    p = subprocess.Popen(*args, **kwargs)
    wait_stdout = None
    wait_stderr = None

    if p.stdout:
        wait_stdout = sys.stdout.attach(p.stdout, read_line=read_line)
    if p.stderr:
        wait_stderr = sys.stderr.attach(p.stderr)

    original_wait = p.wait
    def wait():
        original_wait()

        if wait_stdout:
            wait_stdout()
        if wait_stderr:
            wait_stderr()

    p.wait = wait

    return p

Response: Executes a command using subprocess. Popen and redirects output to AETROS and stdout API calls.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def context():
    job = JobBackend()

    offline = False
    if '1' == os.getenv('AETROS_OFFLINE', ''):
        offline = True

    if os.getenv('AETROS_JOB_ID'):
        job.load(os.getenv('AETROS_JOB_ID'))
        if not offline:
            job.connect()
    else:
        job.create()
        if not offline:
            job.connect()

    job.start(offline=offline)

    return job

Response:
Returns a JobBackend instance which connects to AETROS Trainer
    based on model name in aetros. yml or environment variable AETROS_MODEL_NAME environment variable.


Instruction:
Write a Python 3 function that can
call by the process when SIGINT is received.

Response:
def on_sigint(self, sig, frame):
        """
        We got SIGINT signal.
        """

        if self.stop_requested or self.stop_requested_force:
            # signal has already been sent or we force a shutdown.
            # handles the keystroke 2x CTRL+C to force an exit.
            self.stop_requested_force = True
            self.logger.warning('Force stopped: ' + str(sig))

            # just kill the process, we don't care about the results
            self.on_force_exit()
            os._exit(1)
            # with force_exit we really close the process, killing it in unknown state
            # self.fail('Force stopped', force_exit=True)
            # return

        if self.is_master_process():
            self.logger.warning('Received signal '+str(sig)+'. Send again to force stop. Stopping ...')
        else:
            self.logger.debug("Got child signal " + str(sig))

        self.stop_requested = True

        # the default SIGINT handle in python is not always installed, so we can't rely on the
        # KeyboardInterrupt exception to be thrown.
        # thread.interrupt_main would call sigint again.
        # the shutdown listener will do the rest like committing rest memory files into Git and closing connections.
        sys.exit(0 if self.in_early_stop else 1)


Instruction:
Given the following Python 3 function, write the documentation
def external_aborted(self, params):
        self.ended = True
        self.running = False

        # When the server sends an abort signal, we really have to close immediately,
        # since for example the job has been already deleted.
        # without touching the git and client any further
        os._exit(1)

Response: This is called when the server sends an external abort signal.


Instruction:
Can you generate the documentation for the following Python 3 function
def external_stop(self, force):

        # only the master processes handles the regular stop signal from the server, sending a SIGINT to
        # all its child (means to us, non-master process)
        if not self.is_master_process():
            if force:
                # make sure even the subprocess dies really on force
                os._exit(1)

            return

        self.logger.warning("Received stop signal by server.")
        if not self.stop_requested_force:
            self.stop_requested_force = force

        raise_sigint()

Response: Stop signal by server.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def step(self, step, total, label='STEP', speed_label='STEPS/S', size=1):

        self.lock.acquire()
        try:
            time_diff = time.time() - self.last_step_time

            if self.last_step > step:
                # it restarted
                self.last_step = 0

            made_steps_since_last_call = step - self.last_step
            self.last_step = step

            self.made_steps_since_last_sync += made_steps_since_last_call
            self.made_steps_size_since_last_sync += made_steps_since_last_call * size

            if time_diff >= 1 or step == total:  # only each second or last batch
                self.set_system_info('step', step, True)
                self.set_system_info('steps', total, True)

                steps_per_second = self.made_steps_since_last_sync / time_diff
                samples_per_second = self.made_steps_size_since_last_sync / time_diff
                self.last_step_time = time.time()

                if size:
                    self.report_speed(samples_per_second)

                epochs_per_second = steps_per_second / total  # all batches
                self.set_system_info('epochsPerSecond', epochs_per_second, True)

                current_epochs = self.current_epoch if self.current_epoch else 1
                total_epochs = self.total_epochs if self.total_epochs else 1

                self.made_steps_since_last_sync = 0
                self.made_steps_size_since_last_sync = 0

                eta = 0
                if step < total:
                    # time to end this epoch
                    if steps_per_second != 0:
                        eta = (total - step) / steps_per_second

                # time until all epochs are done
                if total_epochs - current_epochs > 0:
                    if epochs_per_second != 0:
                        eta += (total_epochs - (current_epochs)) / epochs_per_second

                self.git.store_file('aetros/job/times/eta.json', simplejson.dumps(eta))

            if label and self.step_label != label:
                self.set_system_info('stepLabel', label, True)
                self.step_label = label

            if speed_label and self.step_speed_label != speed_label:
                self.set_system_info('stepSpeedLabel', speed_label, True)
                self.step_speed_label = speed_label
        finally:
            self.lock.release()

Response: Increase the step indicator.


Instruction:
Can you create a Python 3 function that
creates a new loss channel.

Response:
def create_loss_channel(self, name='loss', xaxis=None, yaxis=None, layout=None):
        """
        :param name: string
        :return: JobLossGraph
        """

        return JobLossChannel(self, name, xaxis, yaxis, layout)


Instruction:
Given the following Python 3 function, write the documentation
def create_channel(self, name, traces=None,
                       main=False, kpi=False, kpiTrace=0, max_optimization=True,
                       type=JobChannel.NUMBER,
                       xaxis=None, yaxis=None, layout=None):
        """
        :param name: str
        :param traces: None|list : per default create a trace based on "name".
        :param main: bool : whether this channel is visible in the job list as column for better comparison.

        :param kpi: bool : whether this channel is the KPI (key performance indicator).
                           Used for hyperparameter optimization. Only one channel can be a kpi. Only first trace used.
        :param kpiTrace: bool : if you have multiple traces, define which is the KPI. 0 based index.

        :param max_optimization: bool : whether the optimization maximizes or minmizes the kpi. Use max_optimization=False to
                                        tell the optimization algorithm that his channel minimizes a kpi, for instance the loss of a model.

        :param type: str : One of JobChannel.NUMBER, JobChannel.TEXT
        :param xaxis: dict
        :param yaxis: dict
        :param layout: dict
        """
        return JobChannel(self, name, traces, main, kpi, kpiTrace, max_optimization, type, xaxis, yaxis, layout)

Response: Create a new channel based on the given parameters.


Instruction:
Create a Python 3 function to
call by the daemon to handle shutdown.

Response:
def on_shutdown(self):
        """
        Shutdown routine. Sets the last progress (done, aborted, failed) and tries to send last logs and git commits.
        Also makes sure the ssh connection is closed (thus, the job marked as offline).

        Is triggered by atexit.register().
        """

        self.in_shutdown = True

        self.logger.debug('on_shutdown, stopped=%s, ended=%s, early_stop=%s, stop_requested=%s'
                          % (str(self.stopped), str(self.ended), str(self.in_early_stop), str(self.stop_requested)))
        if self.stopped or self.ended:
            # make really sure, ssh connection closed
            self.client.close()
            return

        if self.in_early_stop:
            self.done()
            return

        if self.stop_requested:
            # when SIGINT has been triggered
            if self.stop_requested_force:
                if not self.is_master_process():
                    # if not master process, we just stop everything. status/progress is set by master
                    self.stop(force_exit=True)
                else:
                    # master process
                    self.fail('Force stopped.', force_exit=True)
            else:
                if not self.is_master_process():
                    # if not master process, we just stop everything. status/progress is set by master
                    self.stop()
                else:
                    # master process
                    self.abort()

            return

        if hasattr(sys, 'last_value'):
            # sys.last_value contains a exception, when there was an uncaught one
            if isinstance(sys.last_value, KeyboardInterrupt):
                # can only happen when KeyboardInterrupt has been raised manually
                # since the one from the default sigint handler will never reach here
                # since we catch the sigint signal and sys.exit() before the default sigint handler
                # is able to raise KeyboardInterrupt
                self.abort()
            else:
                self.fail(type(sys.last_value).__name__ + ': ' + str(sys.last_value))

        elif self.running:
            self.done()


Instruction:
Given the following Python 3 function, write the documentation
def fail(self, message=None, force_exit=False):
        global last_exit_code

        if not last_exit_code:
            last_exit_code = 1

        with self.git.batch_commit('FAILED'):
            self.set_status('FAILED', add_section=False)
            self.git.commit_json_file('FAIL_MESSAGE', 'aetros/job/crash/error', str(message) if message else '')
            if isinstance(sys.stderr, GeneralLogger):
                self.git.commit_json_file('FAIL_MESSAGE_LAST_LOG', 'aetros/job/crash/last_message', sys.stderr.last_messages)

        self.logger.debug('Crash report stored in commit ' + self.git.get_head_commit())
        self.stop(JOB_STATUS.PROGRESS_STATUS_FAILED, force_exit=force_exit)

Response: Mark the job as failed saves the given error message and force exists the process when force_exit = True.


Instruction:
Make a summary of the following Python 3 code
def write_log(self, message):
        if self.stream_log and not self.ended:
            # points to the Git stream write
            self.stream_log.write(message)
            return True

Response: Write a message to the Git log.


Instruction:
How would you implement a function in Python 3 that
sets an arbitrary status visible in the big wheel of the job view.

Response:
def set_status(self, status, add_section=True):
        """
        Set an arbitrary status, visible in the big wheel of the job view.
        """
        status = str(status)

        if add_section:
            self.section(status)

        self.job_add_status('status', status)


Instruction:
Here you have a function in Python 3, explain what it does
def create(self, create_info=None, hyperparameter=None, server='local', insights=False):
        if not create_info:
            create_info = {
                'server': server,
                'config': {
                    'insights': insights,
                    'command': ' '.join(sys.argv)
                }
            }
            config = find_config(self.config_path, logger=self.logger)

            if not config['model']:
                raise Exception('AETROS config file (aetros.yml) not found.')

            # first transform simple format in the full definition with parameter types
            # (string, number, group, choice_group, etc)
            full_hyperparameters = lose_parameters_to_full(config['parameters'])

            # now extract hyperparameters from full definition, and overwrite stuff using
            # incoming_hyperparameter if available
            hyperparameter = extract_parameters(full_hyperparameters, hyperparameter)
            create_info['config']['parameters'] = hyperparameter

        self.job = create_info

        if 'server' not in self.job and server:
            # setting this disables server assignment
            self.job['server'] = server

        self.job['optimization'] = None
        self.job['type'] = 'custom'

        if 'parameters' not in self.job['config']:
            self.job['config']['parameters'] = {}

        if 'insights' not in self.job['config']:
            self.job['config']['insights'] = insights

        self.job['created'] = time.time()
        self.git.create_job_id(self.job)

        self.logger.debug("Job created with Git ref " + self.git.ref_head)

        return self.job_id

Response: Creates a new job in git and pushes it to the local store.


Instruction:
Can you write a function in Python 3 where it
reads hyperparameter from job configuration.

Response:
def get_parameter(self, path, default=None, return_group=False):
        """
        Reads hyperparameter from job configuration. If nothing found use given default.

        :param path: str
        :param default: *
        :param return_group: If true and path is a choice_group, we return the dict instead of the group name.
        :return: *
        """
        value = read_parameter_by_path(self.job['config']['parameters'], path, return_group)

        if value is None:
            return default

        return value


Instruction:
Can you generate a brief explanation for the following Python 3 code
def load(self, job_id):
        self.git.read_job(job_id, checkout=self.is_master_process())
        self.load_job_from_ref()

Response: Load a job into index and work - tree restarts its ref and sets as current.


Instruction:
Explain what the following Python 3 code does
def load_job_from_ref(self):
        if not self.job_id:
            raise Exception('Job not loaded yet. Use load(id) first.')

        if not os.path.exists(self.git.work_tree + '/aetros/job.json'):
            raise Exception('Could not load aetros/job.json from git repository. Make sure you have created the job correctly.')

        with open(self.git.work_tree + '/aetros/job.json') as f:
            self.job = simplejson.loads(f.read(), object_pairs_hook=collections.OrderedDict)

        if not self.job:
            raise Exception('Could not parse aetros/job.json from git repository. Make sure you have created the job correctly.')

        self.logger.debug('job: ' + str(self.job))

Response: Loads the job. json into self. job


Instruction:
How would you code a function in Python 3 to
return a JobModel instance with current loaded job data attached.

Response:
def get_job_model(self):
        """
        Returns a new JobModel instance with current loaded job data attached.
        :return: JobModel
        """
        if not self.job:
            raise Exception('Job not loaded yet. Use load(id) first.')

        return JobModel(self.job_id, self.job, self.home_config['storage_dir'])


Instruction:
Write a Python 3 script for
listing all files in the working directory.

Response:
def file_list(self):
        """
        Lists all files in the working directory.
        """
        blacklist = ['.git', 'aetros']
        working_tree = self.git.work_tree

        def recursive(path='.'):
            if os.path.basename(path) in blacklist:
                return 0, 0

            if os.path.isdir(path):
                files = []
                for file in os.listdir(path):
                    if path and path != '.':
                        file = path + '/' + file

                    added_files = recursive(file)
                    files += added_files

                return files
            else:
                if path.endswith('.pyc'):
                    return []

                if is_ignored(path, self.config['ignore']):
                    return []

                return [os.path.relpath(path, working_tree)]

        return recursive(working_tree)


Instruction:
Can you write a function in Python 3 where it
adds all files in a working_tree to the list of files in the working_tree.

Response:
def add_files(self, working_tree, report=False):
        """
        Commits all files from limited in aetros.yml. `files` is a whitelist, `exclude_files` is a blacklist.
        If both are empty, we commit all files smaller than 10MB.
        :return:
        """
        blacklist = ['.git']

        def add_resursiv(path = '.', report=report):
            if os.path.basename(path) in blacklist:
                return 0, 0

            if working_tree + '/aetros' == path:
                # ignore in work_tree the folder ./aetros/, as it could be
                # that we checked out a job and start it again.
                return 0, 0

            if os.path.isdir(path):
                files = 0
                size = 0
                for file in os.listdir(path):
                    if path and path != '.':
                        file = path + '/' + file

                    added_files, added_size = add_resursiv(file)
                    files += added_files
                    size += added_size

                return files, size
            else:
                if path.endswith('.pyc'):
                    return 0, 0

                relative_path = os.path.relpath(path, working_tree)

                if is_ignored(relative_path, self.config['ignore']):
                    return 0, 0

                self.logger.debug("added file to job " + relative_path)
                if report:
                    print("Added job file: " + relative_path)

                self.git.add_file_path_in_work_tree(path, working_tree, verbose=False)

                return 1, os.path.getsize(path)

        return add_resursiv(working_tree, report=report)


Instruction:
Given the following Python 3 function, write the documentation
def add_embedding_word2vec(self, x, path, dimensions=None, header_with_dimensions=True):
        if path.endswith('.txt'):
            if not os.path.exists(path):
                raise Exception("Given word2vec file does not exist: " + path)

            f = open(path, 'r')

            if not header_with_dimensions and not dimensions:
                raise Exception('Either the word2vec file should contain the dimensions as header or it needs to be'
                                'specified manually using dimensions=[x,y] argument.')

            if header_with_dimensions:
                line = f.readline()
                if ' ' not in line:
                    raise Exception('Given word2vec file should have in first line the dimensions, e.g.: 1000 200')
                dimensions = np.fromstring(line, dtype=np.uint, sep=' ').tolist()

            labels = ''
            vectors = ''
            line_pos = 1 if header_with_dimensions else 0

            if len(dimensions) != 2:
                raise Exception('dimensions invalid shape. e.g. [200, 32] => 200 rows, 32 cols.')

            for line in iter(f.readline, ''):
                line_pos += 1
                space_pos = line.find(' ')
                if -1 == space_pos:
                    message = 'Given word2vec does not have correct format in line ' + str(line_pos)
                    message += '\nGot: ' + str(line)
                    raise Exception(message)

                labels += line[:space_pos] + '\n'
                vectors += line[space_pos+1:] + ' '

            vectors = np.fromstring(vectors, dtype=np.float32, sep=' ').tobytes()
        else:
            raise Exception("Given word2vec is not a .txt file. Other file formats are not supported.")

        info = {
            'dimensions': dimensions
        }

        name = os.path.basename(path)
        self._ensure_insight(x)
        remote_path = 'aetros/job/insight/'+str(x)+'/embedding/'
        with self.git.batch_commit('INSIGHT_EMBEDDING ' + str(x)):
            self.git.commit_file('WORD2VEC', remote_path + name + '/tensor.bytes', vectors)
            self.git.commit_file('WORD2VEC', remote_path + name + '/metadata.tsv', labels)
            self.git.commit_file('WORD2VEC', remote_path + name + '/info.json', simplejson.dumps(info))

Response: Add an embedding word2vec file to the database.


Instruction:
Write a Python 3 function for
adding a new embedding to the list of images and vectors.

Response:
def add_embedding_path(self, x, dimensions, vectors_path, metadata=None, image_shape=None, image=None):
        """
        Adds a new embedding with optional metadata.

        Example how to generate vectors based on 2D numpy array:

        # 4 vectors, each size of 3
        vectors = [
            [2.3, 4.0, 33],
            [2.4, 4.2, 44],
            [2.5, 3.9, 34],
            [5.5, 200.2, 66]
        ]

        metadata = [
            # header, only necessary when more then on column
            # can be anything.
            ['label', 'count'],

            # for each vector from above an entry.
            ['red', '123'],
            ['white', '143'],
            ['yellow', '344'],
            ['house', '24'],
        ]

        numpy.array(vectors, dtype=numpy.float32).tofile('vectors.bytes')
        numpy.savetxt('metadata.tsv', numpy.array(metadata), delimiter='\t', fmt='%s')

        job.add_embedding_path([4, 3], 'vectors.bytes', 'metadata.tsv')

        Metadata format example:

        Label\tCount\n
        red\t4\n
        yellow\t6\n

        :param x: The x axis of the insights.
        :param dimensions: 2D List of dimension, e.g [200, 20], means 200 vectors and each vector has size of 20.

        :param vectors_path: A path to a floats64 bytes file, no separators, sum(dimensions)*floats64 long.
                       Example: If dimensions [200, 20] then the tensor file has 200*20 float32 bytes in it

        :param metadata: A TSV file. If only one column long (=no tab separator per line), then there's no need for a header.
                         If you have more than one column, use the first line as header.

        :param image_shape: Size of the image of each vector.
        :param image: Path to an image sprite.
        :return:
        """
        if not os.path.exists(vectors_path):
            raise Exception("Given embedding vectors file does not exist: " + vectors_path)

        if metadata and not os.path.exists(metadata):
            raise Exception("Given embedding metadata file does not exist: " + metadata)

        name = os.path.basename(vectors_path)
        self._ensure_insight(x)
        remote_path = 'aetros/job/insight/'+str(x)+'/embedding/'

        info = {
            'dimensions': dimensions,
            'image_shape': image_shape,
            'image': os.path.basename(image) if image else None,
        }

        with self.git.lock_write():
            self.git.add_file_path(remote_path + name + '/tensor.bytes', vectors_path)
            self.git.add_file_path(remote_path + name + '/metadata.tsv', metadata)
            self.git.add_file(remote_path + name + '/info.json', simplejson.dumps(info))

            if image:
                self.git.add_file(remote_path + name + '/' + os.path.basename(image), image)

            self.git.commit_index('INSIGHT_EMBEDDING ' + str(x))


Instruction:
Write a Python 3 function that can
parse sane directory output usually ls - l

Response:
def split_file_info(fileinfo):
    """ Parse sane directory output usually ls -l
        Adapted from https://gist.github.com/tobiasoberrauch/2942716
    """
    current_year = datetime.datetime.now().strftime('%Y')
    files = []
    for line in fileinfo:
        parts = re.split(
            r'^([\-dbclps])' +                  # Directory flag [1]
            r'([\-rwxs]{9})\s+' +               # Permissions [2]
            r'(\d+)\s+' +                       # Number of items [3]
            r'([a-zA-Z0-9_-]+)\s+' +            # File owner [4]
            r'([a-zA-Z0-9_-]+)\s+' +            # File group [5]
            r'(\d+)\s+' +                       # File size in bytes [6]
            r'(\w{3}\s+\d{1,2})\s+' +           # 3-char month and 1/2-char day of the month [7]
            r'(\d{1,2}:\d{1,2}|\d{4})\s+' +     # Time or year (need to check conditions) [+= 7]
            r'(.+)$',                           # File/directory name [8]
            line
        )

        date = parts[7]
        time = parts[8] if ':' in parts[8] else '00:00'
        year = parts[8] if ':' not in parts[8] else current_year
        dt_obj = parser.parse("%s %s %s" % (date, year, time))

        files.append({
            'directory': parts[1],
            'perms': parts[2],
            'items': parts[3],
            'owner': parts[4],
            'group': parts[5],
            'size': int(parts[6]),
            'date': date,
            'time': time,
            'year': year,
            'name': parts[9],
            'datetime': dt_obj
        })
    return files


Instruction:
Can you tell what is the following Python 3 function doing
def get(self, remote, local=None):
        if isinstance(local, file_type):  # open file, leave open
            local_file = local
        elif local is None:  # return string
            local_file = buffer_type()
        else:  # path to file, open, write/close return None
            local_file = open(local, 'wb')

        self.conn.retrbinary("RETR %s" % remote, local_file.write)

        if isinstance(local, file_type):
            pass
        elif local is None:
            contents = local_file.getvalue()
            local_file.close()
            return contents
        else:
            local_file.close()

        return None

Response: Gets the file from FTP server


Instruction:
Write a Python 3 function that can
put a local file on the FTP server and return the size of the file.

Response:
def put(self, local, remote, contents=None, quiet=False):
        """ Puts a local file (or contents) on to the FTP server

            local can be:
                a string: path to inpit file
                a file: opened for reading
                None: contents are pushed
        """
        remote_dir = os.path.dirname(remote)
        remote_file = os.path.basename(local)\
            if remote.endswith('/') else os.path.basename(remote)

        if contents:
            # local is ignored if contents is set
            local_file = buffer_type(contents)
        elif isinstance(local, file_type):
            local_file = local
        else:
            local_file = open(local, 'rb')
        current = self.conn.pwd()
        self.descend(remote_dir, force=True)

        size = 0
        try:
            self.conn.storbinary('STOR %s' % remote_file, local_file)
            size = self.conn.size(remote_file)
        except:
            if not quiet:
                raise
        finally:
            local_file.close()
            self.conn.cwd(current)
        return size


Instruction:
Can you tell what is the following Python 3 function doing
def list(self, remote='.', extra=False, remove_relative_paths=False):
        if extra:
            self.tmp_output = []
            self.conn.dir(remote, self._collector)
            directory_list = split_file_info(self.tmp_output)
        else:
            directory_list = self.conn.nlst(remote)

        if remove_relative_paths:
            return list(filter(self.is_not_relative_path, directory_list))

        return directory_list

Response: Return a list of all the files in the local directory.


Instruction:
Write a Python 3 function that can
descend the current directory with the remote directory.

Response:
def descend(self, remote, force=False):
        """ Descend, possibly creating directories as needed """
        remote_dirs = remote.split('/')
        for directory in remote_dirs:
            try:
                self.conn.cwd(directory)
            except Exception:
                if force:
                    self.conn.mkd(directory)
                    self.conn.cwd(directory)
        return self.conn.pwd()


Instruction:
Explain what the following Python 3 code does
def delete(self, remote):
        try:
            self.conn.delete(remote)
        except Exception:
            return False
        else:
            return True

Response: Delete a file from server


Instruction:
How would you implement a function in Python 3 that
changes working directory on server

Response:
def cd(self, remote):
        """ Change working directory on server """
        try:
            self.conn.cwd(remote)
        except Exception:
            return False
        else:
            return self.pwd()


Instruction:
How would you code a function in Python 3 to
start a job with all logging of a job_id

Response:
def start(logger, full_id, fetch=True, env=None, volumes=None, cpus=None, memory=None, gpu_devices=None, offline=False):
    """
    Starts the job with all logging of a job_id
    """

    owner, name, id = unpack_full_job_id(full_id)

    if isinstance(sys.stdout, GeneralLogger):
        # we don't want to have stuff written to stdout before in job's log
        sys.stdout.clear_buffer()

    job_backend = JobBackend(model_name=owner + '/' + name)

    if fetch:
        job_backend.fetch(id)

    job_backend.restart(id)
    job_backend.start(collect_system=False, offline=offline)
    job_backend.set_status('PREPARE', add_section=False)

    job = job_backend.get_job_model()

    if not cpus:
        cpus = job.get_cpu()

    if not memory:
        memory = job.get_memory()

    if not gpu_devices and job.get_gpu():
        # if requested 2 GPUs and we have 3 GPUs with id [0,1,2], gpus should be [0,1]
        gpu_devices = []
        for i in range(0, job.get_gpu()):
            gpu_devices.append(i)

    start_command(logger, job_backend, env, volumes, cpus=cpus, memory=memory, gpu_devices=gpu_devices, offline=offline)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def fromimage(im, flatten=False, mode=None):
    if not Image.isImageType(im):
        raise TypeError("Input is not a PIL image.")

    if mode is not None:
        if mode != im.mode:
            im = im.convert(mode)
    elif im.mode == 'P':
        # Mode 'P' means there is an indexed "palette".  If we leave the mode
        # as 'P', then when we do `a = array(im)` below, `a` will be a 2-D
        # containing the indices into the palette, and not a 3-D array
        # containing the RGB or RGBA values.
        if 'transparency' in im.info:
            im = im.convert('RGBA')
        else:
            im = im.convert('RGB')

    if flatten:
        im = im.convert('F')
    elif im.mode == '1':
        # Workaround for crash in PIL. When im is 1-bit, the call array(im)
        # can cause a seg. fault, or generate garbage. See
        # https://github.com/scipy/scipy/issues/2138 and
        # https://github.com/python-pillow/Pillow/issues/350.
        #
        # This converts im from a 1-bit image to an 8-bit image.
        im = im.convert('L')

    a = array(im)
    return a

Response: Returns a copy of a PIL image as a numpy array.


Instruction:
Make a summary of the following Python 3 code
def toimage(arr, high=255, low=0, cmin=None, cmax=None, pal=None,
            mode=None, channel_axis=None):
    """Takes a numpy array and returns a PIL image.

    The mode of the PIL image depends on the array shape and the `pal` and
    `mode` keywords.

    For 2-D arrays, if `pal` is a valid (N,3) byte-array giving the RGB values
    (from 0 to 255) then ``mode='P'``, otherwise ``mode='L'``, unless mode
    is given as 'F' or 'I' in which case a float and/or integer array is made.

    Notes
    -----
    For 3-D arrays, the `channel_axis` argument tells which dimension of the
    array holds the channel data.

    For 3-D arrays if one of the dimensions is 3, the mode is 'RGB'
    by default or 'YCbCr' if selected.

    The numpy array must be either 2 dimensional or 3 dimensional.

    """
    data = asarray(arr)
    if iscomplexobj(data):
        raise ValueError("Cannot convert a complex-valued array.")
    shape = list(data.shape)
    valid = len(shape) == 2 or ((len(shape) == 3) and
                                ((3 in shape) or (4 in shape)))
    if not valid:
        raise ValueError("'arr' does not have a suitable array shape for "
                         "any mode.")
    if len(shape) == 2:
        shape = (shape[1], shape[0])  # columns show up first
        if mode == 'F':
            data32 = data.astype(numpy.float32)
            image = Image.frombytes(mode, shape, data32.tostring())
            return image
        if mode in [None, 'L', 'P']:
            bytedata = bytescale(data, high=high, low=low,
                                 cmin=cmin, cmax=cmax)
            image = Image.frombytes('L', shape, bytedata.tostring())
            if pal is not None:
                image.putpalette(asarray(pal, dtype=uint8).tostring())
                # Becomes a mode='P' automagically.
            elif mode == 'P':  # default gray-scale
                pal = (arange(0, 256, 1, dtype=uint8)[:, newaxis] *
                       ones((3,), dtype=uint8)[newaxis, :])
                image.putpalette(asarray(pal, dtype=uint8).tostring())
            return image
        if mode == '1':  # high input gives threshold for 1
            bytedata = (data > high)
            image = Image.frombytes('1', shape, bytedata.tostring())
            return image
        if cmin is None:
            cmin = amin(ravel(data))
        if cmax is None:
            cmax = amax(ravel(data))
        data = (data*1.0 - cmin)*(high - low)/(cmax - cmin) + low
        if mode == 'I':
            data32 = data.astype(numpy.uint32)
            image = Image.frombytes(mode, shape, data32.tostring())
        else:
            raise ValueError(_errstr)
        return image

    # if here then 3-d array with a 3 or a 4 in the shape length.
    # Check for 3 in datacube shape --- 'RGB' or 'YCbCr'
    if channel_axis is None:
        if (3 in shape):
            ca = numpy.flatnonzero(asarray(shape) == 3)[0]
        else:
            ca = numpy.flatnonzero(asarray(shape) == 4)
            if len(ca):
                ca = ca[0]
            else:
                raise ValueError("Could not find channel dimension.")
    else:
        ca = channel_axis

    numch = shape[ca]
    if numch not in [3, 4]:
        raise ValueError("Channel axis dimension is not valid.")

    bytedata = bytescale(data, high=high, low=low, cmin=cmin, cmax=cmax)
    if ca == 2:
        strdata = bytedata.tostring()
        shape = (shape[1], shape[0])
    elif ca == 1:
        strdata = transpose(bytedata, (0, 2, 1)).tostring()
        shape = (shape[2], shape[0])
    elif ca == 0:
        strdata = transpose(bytedata, (1, 2, 0)).tostring()
        shape = (shape[2], shape[1])
    if mode is None:
        if numch == 3:
            mode = 'RGB'
        else:
            mode = 'RGBA'

    if mode not in ['RGB', 'RGBA', 'YCbCr', 'CMYK']:
        raise ValueError(_errstr)

    if mode in ['RGB', 'YCbCr']:
        if numch != 3:
            raise ValueError("Invalid array shape for mode.")
    if mode in ['RGBA', 'CMYK']:
        if numch != 4:
            raise ValueError("Invalid array shape for mode.")

    # Here we know data and mode is correct
    image = Image.frombytes(mode, shape, strdata)
    return image

Response: Takes a numpy array and returns a PIL image.


Instruction:
Here you have a function in Python 3, explain what it does
def imresize(arr, size, interp='bilinear', mode=None):
    im = toimage(arr, mode=mode)
    ts = type(size)
    if issubdtype(ts, int):
        percent = size / 100.0
        size = tuple((array(im.size)*percent).astype(int))
    elif issubdtype(type(size), float):
        size = tuple((array(im.size)*size).astype(int))
    else:
        size = (size[1], size[0])
    func = {'nearest': 0, 'lanczos': 1, 'bilinear': 2, 'bicubic': 3, 'cubic': 3}
    imnew = im.resize(size, resample=func[interp])
    return fromimage(imnew)

Response:
Resize an image.

    Parameters
    ----------
    arr : ndarray
        The array of image to be resized.

    size : int, float or tuple
        * int   - Percentage of current size.
        * float - Fraction of current size.
        * tuple - Size of the output image.

    interp : str, optional
        Interpolation to use for re-sizing ('nearest', 'lanczos', 'bilinear', 'bicubic'
        or 'cubic').

    mode : str, optional
        The PIL image mode ('P', 'L', etc.) to convert `arr` before resizing.

    Returns
    -------
    imresize : ndarray
        The resized array of image.

    See Also
    --------
    toimage : Implicitly used to convert `arr` according to `mode`.
    scipy.ndimage.zoom : More generic implementation that does not use PIL.


Instruction:
How would you implement a function in Python 3 that
connects to a specific channel.

Response:
def connect(self, channel):
        """
        In the write-thread we detect that no connection is living anymore and try always again.
        Up to the 3 connection try, we report to user. We keep trying but in silence.
        Also, when more than 10 connection tries are detected, we delay extra 15 seconds.
        """
        if self.connection_tries > 10:
            time.sleep(10)

        if self.in_connecting[channel]:
            return False

        self.in_connecting[channel] = True

        self.logger.debug('[%s] Wanna connect ...' % (channel, ))

        try:
            if self.is_connected(channel) or self.online is False:
                if self.is_connected(channel):
                    self.logger.debug('[%s] Already connected' % (channel, ))
                if self.online is False:
                    self.logger.debug('[%s] self.online=False' % (channel, ))

                return True

            self.channel_lock[channel].acquire()

            self.connected[channel] = None
            self.registered[channel] = None
            self.ssh_stream[channel] = False
            self.ssh_channel[channel] = False
            messages = None
            stderrdata = ''

            try:
                if not self.ssh_stream[channel]:
                    self.logger.debug('[%s] Open ssh connection' % (channel, ))
                    self.ssh_stream[channel] = create_ssh_stream(self.config, exit_on_failure=False)

                self.logger.debug('[%s] open channel' % (channel, ))

                self.ssh_channel[channel] = self.ssh_stream[channel].get_transport().open_session()
                self.ssh_channel[channel].exec_command('stream')
            except (KeyboardInterrupt, SystemExit):
                raise
            except Exception as e:
                self.connected[channel] = False
                self.registered[channel] = False
                self.logger.debug('[%s] connection failed: %s'  % (channel, str(e)))
                return False
            finally:
                self.channel_lock[channel].release()

            if self.ssh_channel[channel]:
                messages = self.wait_for_at_least_one_message(channel)

            if not messages:
                stderrdata = self.ssh_channel[channel].recv_stderr().decode("utf-8").strip()
                self.connected[channel] = False
                self.registered[channel] = False
            else:
                self.logger.debug('[%s] opened and received %d messages' % (channel, len(messages)))
                self.connected[channel] = True
                self.registered[channel] = self.on_connect(self.was_connected_once[channel], channel)
                self.connected_since[channel] = time.time()

                if channel == '' and self.registered[channel] and self.was_connected_once[channel]:
                    self.logger.info("Successfully reconnected.")

            if not self.registered[channel]:
                # make sure to close channel and connection first
                try:
                    self.ssh_channel[channel] and self.ssh_channel[channel].close()
                except: pass

                try:
                    self.ssh_stream[channel] and self.ssh_stream[channel].close()
                except: pass

                self.logger.debug("[%s] Client: registration failed. stderrdata: %s" % (channel, stderrdata))
                self.connected[channel] = False

                try:
                    self.logger.debug('[%s] Client: ssh_tream close due to registration failure' % (channel, ))
                    self.ssh_stream[channel].close()
                except (KeyboardInterrupt, SystemExit):
                    raise

                self.connection_tries += 1
                if not self.was_connected_once[channel] and self.go_offline_on_first_failed_attempt:
                    # initial try needs to be online, otherwise we go offline
                    self.go_offline()

                if stderrdata:
                    if 'Connection refused' not in stderrdata and 'Permission denied' not in stderrdata:
                        self.logger.error(stderrdata)

                if 'Permission denied' in stderrdata:
                    if self.connection_tries < 3:
                        self.logger.warning("Access denied. Did you setup your SSH public key correctly "
                                            "and saved it in your AETROS Trainer user account?")

                    self.close()
                    sys.exit(1)

                self.connection_error(channel, "Connection error during connecting to %s: %s" % (self.host, str(stderrdata)))
            else:
                self.was_connected_once[channel] = True

        except Exception as error:
            self.connection_error(channel, error)
        finally:
            self.in_connecting[channel] = False

        return self.is_connected(channel)


Instruction:
Can you tell what is the following Python 3 function doing
def _end_channel(self, channel):
        self.stop_on_empty_queue[channel] = True

        # by joining the we wait until its loop finishes.
        # it won't loop forever since we've set self.stop_on_empty_queue=True
        write_thread = self.thread_write_instances[channel]

        thread_join_non_blocking(write_thread)

Response: End the channel and wait for the write thread to finish.


Instruction:
Explain what the following Python 3 code does
def wait_sending_last_messages(self):
        if self.active and self.online is not False:
            self.logger.debug("client sends last %s messages ..."
                              % ([str(i) + ':' + str(len(x)) for i, x in six.iteritems(self.queues)],))

            for channel, messages in six.iteritems(self.queues):
                for idx, message in enumerate(messages):
                    self.logger.debug("[%s] %d: %s" % (channel, idx, str(message)[0:120]))

            # send all missing messages

            # by joining we wait until its loop finish.
            # it won't loop forever since we've set self.stop_on_empty_queue=True
            for channel in six.iterkeys(self.ssh_channel):
                if channel != '':
                    self._end_channel(channel)

            # last is control channel
            self._end_channel('')

Response: Send all messages to the last available channel.


Instruction:
Write a Python 3 function for
waiting until all queues of channels are empty.

Response:
def wait_until_queue_empty(self, channels, report=True, clear_end=True):
        """
        Waits until all queues of channels are empty.
        """
        state = {'message': ''}

        self.logger.debug("wait_until_queue_empty: report=%s %s"
                          % (str(report), str([channel+':'+str(len(self.queues[channel])) for channel in channels]), ))
        queues = []
        for channel in channels:
            queues += self.queues[channel][:]

        def print_progress():
            if report:
                self.logger.debug("all_empty=%s" % (str(all_empty),))

                sys.__stderr__.write('\b' * len(state['message']))
                sys.__stderr__.write("\033[K")

                state['message'] = "%.2f kB/s // %.2fkB of %.2fkB // %.2f%%" \
                          % (self.bytes_speed / 1024, self.bytes_sent / 1024, self.bytes_total / 1024,
                            (self.bytes_sent / self.bytes_total * 100) if self.bytes_total else 0)

                sys.__stderr__.write(state['message'])
                sys.__stderr__.flush()

        while True:
            all_empty = all(m['_sent'] for m in queues)

            print_progress()

            if all_empty:
                break

            time.sleep(0.2)

        print_progress()

        if report and clear_end:
            sys.__stderr__.write('\b' * len(state['message']))
            sys.__stderr__.write("\033[K")
            sys.__stderr__.flush()


Instruction:
Create a Python 3 function to
read until we receive at least one message. Return all found messages.

Response:
def wait_for_at_least_one_message(self, channel):
        """
        Reads until we receive at least one message we can unpack. Return all found messages.
        """

        unpacker = msgpack.Unpacker(encoding='utf-8')

        while True:
            try:
                start = time.time()
                chunk = self.ssh_channel[channel].recv(1024)
                end = time.time()
                self.read_speeds.append( len(chunk) / (end-start) )
                if len(self.read_speeds) > 20:
                    self.read_speeds = self.read_speeds[10:]

                if chunk == b'':
                    # happens only when connection broke. If nothing is to be received, it hangs instead.
                    self.connection_error(channel, 'Connection broken w')
                    return False
            except Exception as error:
                self.connection_error(channel, error)
                raise

            unpacker.feed(chunk)

            messages = [m for m in unpacker]
            if messages:
                return messages


Instruction:
Can you tell what is the following Python 3 function doing
def read(self, channel):

        if not self.ssh_channel[channel].recv_ready():
            return

        try:
            start = time.time()
            chunk = self.ssh_channel[channel].recv(1024)
            end = time.time()

            self.read_speeds.append(len(chunk) / (end-start))
            if len(self.read_speeds) > 20:
                self.read_speeds = self.read_speeds[10:]

        except Exception as error:
            self.connection_error(channel, error)
            raise

        if chunk == b'':
            # socket connection broken
            self.connection_error(channel, 'Connection broken')
            return None

        # self.read_buffer.seek(0, 2) #make sure we write at the end
        self.read_unpacker.feed(chunk)

        # self.read_buffer.seek(0)
        messages = [m for m in self.read_unpacker]

        return messages if messages else None

Response: Reads from the socket and returns the message.


Instruction:
How would you implement a function in Python 3 that
raises the SIGINT signal in the current process and all sub - processes.

Response:
def raise_sigint():
    """
    Raising the SIGINT signal in the current process and all sub-processes.

    os.kill() only issues a signal in the current process (without subprocesses).
    CTRL+C on the console sends the signal to the process group (which we need).
    """
    if hasattr(signal, 'CTRL_C_EVENT'):
        # windows. Need CTRL_C_EVENT to raise the signal in the whole process group
        os.kill(os.getpid(), signal.CTRL_C_EVENT)
    else:
        # unix.
        pgid = os.getpgid(os.getpid())
        if pgid == 1:
            os.kill(os.getpid(), signal.SIGINT)
        else:
            os.killpg(os.getpgid(os.getpid()), signal.SIGINT)


Instruction:
How would you explain what the following Python 3 function does
def human_size(size_bytes, precision=0):
    if size_bytes == 1:
        # because I really hate unnecessary plurals
        return "1 byte"

    suffixes_table = [('bytes',0),('KB',0),('MB',1),('GB',2),('TB',2), ('PB',2)]

    num = float(size_bytes)
    for suffix, precision in suffixes_table:
        if num < 1024.0:
            break
        num /= 1024.0

    if precision == 0:
        formatted_size = "%d" % num
    else:
        formatted_size = str(round(num, ndigits=precision))

    return "%s %s" % (formatted_size, suffix)

Response: Format a size in bytes into a human readable file size e. g. 1 byte KB GB TB PB


Instruction:
Write a Python 3 function that can
convert an array to an image.

Response:
def array_to_img(x, scale=True):
    """
    x should be shape (channels, width, height)
    """
    from PIL import Image
    if x.ndim != 3:
        raise Exception('Unsupported shape : ', str(x.shape), '. Need (channels, width, height)')
    if scale:
        x += max(-np.min(x), 0)
        x /= np.max(x)
        x *= 255
    if x.shape[0] == 3:
        # RGB
        if x.dtype != 'uint8':
            x = x.astype('uint8')
        return Image.fromarray(x.astype('uint8'), 'RGB')
    elif x.shape[0] == 1:
        # grayscale
        if x.dtype != 'uint8':
            x = x.astype('uint8')
        return Image.fromarray(x.reshape(x.shape[1], x.shape[2]), 'L')
    else:
        raise Exception('Unsupported channel number: ', x.shape[0])


Instruction:
Explain what the following Python 3 code does
def set_generator_validation_nb(self, number):

        self.nb_val_samples = number
        diff_to_batch = number % self.get_batch_size()
        if diff_to_batch > 0:
            self.nb_val_samples += self.get_batch_size() - diff_to_batch

        import keras
        if '1' != keras.__version__[0]:
            self.nb_val_samples = self.nb_val_samples // self.get_batch_size()

Response:
Sets self. nb_val_samples which is used in model. fit if input is a generator
       .