Instruction:
Given the following Python 3 function, write the documentation
def set_default(self, channels=None):
        '''Setting default voltage
        '''
        if not channels:
            channels = self._ch_cal.keys()
        for channel in channels:
            self.set_voltage(channel, self._ch_cal[channel]['default'], unit='V')

Response: Sets default voltage for all the specified channels.


Instruction:
Can you generate the documentation for the following Python 3 function
def get_voltage(self, channel, unit='V'):
        '''Reading voltage
        '''
        kwargs = self._ch_map[channel]['ADCV']
        voltage_raw = self._get_adc_value(**kwargs)

        voltage = (voltage_raw - self._ch_cal[channel]['ADCV']['offset']) / self._ch_cal[channel]['ADCV']['gain']

        if unit == 'raw':
            return voltage_raw
        elif unit == 'V':
            return voltage
        elif unit == 'mV':
            return voltage * 1000
        else:
            raise TypeError("Invalid unit type.")

Response: Reads voltage from the ADC and returns it in the specified unit.


Instruction:
Can you tell what is the following Python 3 function doing
def get_current(self, channel, unit='A'):
        '''Reading current
        '''
        kwargs = self._ch_map[channel]['ADCI']
        current_raw = self._get_adc_value(**kwargs)
        voltage = self.get_voltage(channel)

        current_raw_iq = current_raw - (self._ch_cal[channel]['ADCI']['iq_offset'] + self._ch_cal[channel]['ADCI']['iq_gain'] * voltage)  # quiescent current (IQ) compensation
        current = (current_raw_iq - self._ch_cal[channel]['ADCI']['offset']) / self._ch_cal[channel]['ADCI']['gain']

        if unit == 'raw':
            return current_raw
        elif unit == 'raw_iq':
            return current_raw_iq
        elif unit == 'A':
            return current
        elif unit == 'mA':
            return current * 1000
        elif unit == 'uA':
            return current * 1000000
        else:
            raise TypeError("Invalid unit type.")

Response: Reads current from the ADC and returns it in the specified unit


Instruction:
Can you write a function in Python 3 where it
reads the EEPROM calibration for power regulators and temperature.

Response:
def read_eeprom_calibration(self, temperature=False):  # use default values for temperature, EEPROM values are usually not calibrated and random
        '''Reading EEPROM calibration for power regulators and temperature
        '''
        header = self.get_format()
        if header == self.HEADER_V1:
            data = self._read_eeprom(self.CAL_DATA_ADDR, size=calcsize(self.CAL_DATA_V1_FORMAT))
            for idx, channel in enumerate(self._ch_cal.iterkeys()):
                ch_data = data[idx * calcsize(self.CAL_DATA_CH_V1_FORMAT):(idx + 1) * calcsize(self.CAL_DATA_CH_V1_FORMAT)]
                values = unpack_from(self.CAL_DATA_CH_V1_FORMAT, ch_data)
                self._ch_cal[channel]['name'] = "".join([c for c in values[0] if (c in string.printable)])  # values[0].strip()
                self._ch_cal[channel]['default'] = values[1]
                self._ch_cal[channel]['ADCI']['gain'] = values[2]
                self._ch_cal[channel]['ADCI']['offset'] = values[3]
                self._ch_cal[channel]['ADCI']['iq_gain'] = values[4]
                self._ch_cal[channel]['ADCI']['iq_offset'] = values[5]
                self._ch_cal[channel]['ADCV']['gain'] = values[6]
                self._ch_cal[channel]['ADCV']['offset'] = values[7]
                self._ch_cal[channel]['DACV']['gain'] = values[8]
                self._ch_cal[channel]['DACV']['offset'] = values[9]
            const_data = data[-calcsize(self.CAL_DATA_CONST_V1_FORMAT):]
            values = unpack_from(self.CAL_DATA_CONST_V1_FORMAT, const_data)
            if temperature:
                for channel in self._ch_cal.keys():
                    self._ch_cal[channel]['VNTC']['B_NTC'] = values[0]
                    self._ch_cal[channel]['VNTC']['R1'] = values[1]
                    self._ch_cal[channel]['VNTC']['R2'] = values[2]
                    self._ch_cal[channel]['VNTC']['R4'] = values[3]
                    self._ch_cal[channel]['VNTC']['R_NTC_25'] = values[4]
                    self._ch_cal[channel]['VNTC']['VREF'] = values[5]
        else:
            raise ValueError('EEPROM data format not supported (header: %s)' % header)


Instruction:
Here you have a function in Python 3, explain what it does
def get_temperature(self, channel, sensor='VNTC'):
        '''Reading temperature
        '''
#         NTC type SEMITEC 103KT1608 http://www.semitec.co.jp/english/products/pdf/KT_Thermistor.pdf
#
#         R_NTC = R_25 * exp(B_NTC * (1/T - 1/T_25))
#
#         R_NTC       measured NTC resistance
#         R_NTC_25    resistance @ 25C
#         B_NTC       temperature coefficient
#         Temperature current temperature (Kelvin)
#         T_25        298,15 K (25C)
#
#         B_NTC       NTC 'b' coefficient, NTC Semitec 103KT1608-1P
#         R_NTC_25    NTC 25C resistance, NTC Semitec 103KT1608-1P
#         R1          resistor value for NTC voltage divider
#         R2          value of R2 in the reference voltage divider
#         R4          value of R4 in the reference voltage divider
#         VREF        supply voltage of the resistor bridge
#

        kwargs = self._ch_map[channel][sensor]
        temp_raw = self._get_adc_value(**kwargs)

        v_adc = ((temp_raw - self._ch_cal.items()[0][1]['ADCV']['offset']) / self._ch_cal.items()[0][1]['ADCV']['gain'])  # voltage, VDDA1
        k = self._ch_cal[channel][sensor]['R4'] / (self._ch_cal[channel][sensor]['R2'] + self._ch_cal[channel][sensor]['R4'])  # reference voltage divider
        r_ntc = self._ch_cal[channel][sensor]['R1'] * (k - v_adc / self._ch_cal[channel][sensor]['VREF']) / (1 - k + v_adc / self._ch_cal[channel][sensor]['VREF'])  # NTC resistance

        return (self._ch_cal[channel][sensor]['B_NTC'] / (log(r_ntc) - log(self._ch_cal[channel][sensor]['R_NTC_25']) + self._ch_cal[channel][sensor]['B_NTC'] / self.T_KELVIN_25)) - self.T_KELVIN_0

Response: Reading temperature in the specified sensor


Instruction:
Write a Python 3 function for
filtering and convert any given data array of any dtype.

Response:
def convert_data_array(arr, filter_func=None, converter_func=None):
    '''Filter and convert any given data array of any dtype.

    Parameters
    ----------
    arr : numpy.array
        Data array of any dtype.
    filter_func : function
        Function that takes array and returns true or false for each item in array.
    converter_func : function
        Function that takes array and returns an array or tuple of arrays.

    Returns
    -------
    array of specified dimension (converter_func) and content (filter_func)
    '''
#     if filter_func != None:
#         if not hasattr(filter_func, '__call__'):
#             raise ValueError('Filter is not callable')
    if filter_func:
        array = arr[filter_func(arr)]  # Indexing with Boolean Arrays
#     if converter_func != None:
#         if not hasattr(converter_func, '__call__'):
#             raise ValueError('Converter is not callable')
    if converter_func:
        arr = converter_func(arr)
    return array


Instruction:
Make a summary of the following Python 3 code
def logical_and(f1, f2):  # function factory
    '''Logical and from functions.

    Parameters
    ----------
    f1, f2 : function
        Function that takes array and returns true or false for each item in array.

    Returns
    -------
    Function

    Examples
    --------
    filter_func=logical_and(is_data_record, is_data_from_channel(4))  # new filter function
    filter_func(array) # array that has Data Records from channel 4
    '''
    def f_and(arr):
        return np.logical_and(f1(arr), f2(arr))
    f_and.__name__ = f1.__name__ + "_and_" + f2.__name__
    return f_and

Response: Logical and from functions.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def logical_or(f1, f2):  # function factory
    '''Logical or from functions.

    Parameters
    ----------
    f1, f2 : function
        Function that takes array and returns true or false for each item in array.

    Returns
    -------
    Function
    '''
    def f_or(arr):
        return np.logical_or(f1(arr), f2(arr))
    f_or.__name__ = f1.__name__ + "_or_" + f2.__name__
    return f_or

Response: Returns a function that returns true or false for each item in array f1 and f2.


Instruction:
Make a summary of the following Python 3 code
def logical_not(f):  # function factory
    '''Logical not from functions.

    Parameters
    ----------
    f1, f2 : function
        Function that takes array and returns true or false for each item in array.

    Returns
    -------
    Function
    '''
    def f_not(arr):
        return np.logical_not(f(arr))
    f_not.__name__ = "not_" + f.__name__
    return f_not

Response: Returns a function that returns true or false for each item in array f


Instruction:
Explain what the following Python 3 code does
def arr_select(value):  # function factory
    '''Selecting array elements by bitwise and comparison to a given value.

    Parameters:
    value : int
        Value to which array elements are compared to.

    Returns:
    array : np.array
    '''
    def f_eq(arr):
        return np.equal(np.bitwise_and(arr, value), value)
    f_eq.__name__ = "arr_bitwise_and_" + str(value)  # or use inspect module: inspect.stack()[0][3]
    return f_eq

Response: Returns a function that returns True if the array elements are equal to a given value.


Instruction:
Explain what the following Python 3 code does
def arr_astype(arr_type):  # function factory
    '''Change dtype of array.

    Parameters:
    arr_type : str, np.dtype
        Character codes (e.g. 'b', '>H'), type strings (e.g. 'i4', 'f8'), Python types (e.g. float, int) and numpy dtypes (e.g. np.uint32) are allowed.

    Returns:
    array : np.array
    '''
    def f_astype(arr):
        return arr.astype(arr_type)
    f_astype.__name__ = "arr_astype_" + str(arr_type)  # or use inspect module: inspect.stack()[0][3]
    return f_astype

Response: Returns a function that can be used to change dtype of array.


Instruction:
Given the following Python 3 function, write the documentation
def send(self, obj):
        assert isinstance(obj, ProtocolBase)
        string = pickle.dumps(obj)
        length = len(string)
        self.sock.sendall(struct.pack("<I", length) + string)

Response: Send a string to the socket.


Instruction:
Can you generate the documentation for the following Python 3 function
def recv(self, blocking=True):
        length = struct.unpack("<I", self.sock.recv(4))[0]
        return self._get_next_obj(length)

Response: Receive the next object from the socket


Instruction:
Can you write a function in Python 3 where it
tries to read a new object from the socket.

Response:
def try_recv(self):
        """Return None immediately if nothing is waiting"""
        try:
            lenstr = self.sock.recv(4, socket.MSG_DONTWAIT)
        except socket.error:
            return None
        if len(lenstr) < 4:
            raise EOFError("Socket closed")
        length = struct.unpack("<I", lenstr)[0]
        return self._get_next_obj(length)


Instruction:
Can you generate the documentation for the following Python 3 function
def _get_next_obj(self, length):
        data = b''
        while len(data) < length:
            data += self.sock.recv(length - len(data))

        return pickle.loads(data)

Response: Get the next object from the socket.


Instruction:
How would you explain what the following Python 3 function does
def get_modules(self, type_name):
        '''Getting modules by type name.

        Parameters
        ----------
        type_name : string
            Type name of the modules to be returned.

        Returns
        -------
        List of modules of given type name else empty list.
        '''
        modules = []
        for module in self:
            if module.__class__.__name__ == type_name:
                modules.append(module)
        return modules

Response: Gets the modules of a given type name.


Instruction:
Write a Python 3 function that can
read response to command and convert it to 16 - bit integer.

Response:
def ask(self, command):
        '''Read response to command and convert it to 16-bit integer.
        Returns : list of values
        '''
        self.write(command)
        time.sleep(0.1)
        return self.read()


Instruction:
Here you have a function in Python 3, explain what it does
def set_enable(self, on, channel=1):
        if isinstance(channel, str):
            cmd = "OPALL %d" % int(on)
        elif isinstance(channel, int):
            cmd = "OP%d %d" % (channel, int(on))
        self.write(cmd)

Response: Enable or disable the active class for this class.


Instruction:
Can you generate the documentation for the following Python 3 function
def get_current(self, channel):
        ret = self.ask("I%dO?" % channel)
        if ret[-1] != "A":
            print("ttiQl355tp.get_current() format error", ret)
            return None
        return float(ret[:-1])

Response: get current time from the ttable


Instruction:
Make a summary of the following Python 3 code
def get_voltage(self, channel):
        ret = self.ask("V%dO?" % channel)
        if ret[-1] != "V":
            print("ttiQl355tp.get_voltage() format error", ret)
            return None
        return float(ret[:-1])

Response: get the voltage of a single channel


Instruction:
Explain what the following Python 3 code does
def get_set_voltage(self, channel):
        ret = self.ask("V%d?" % channel)
        if ret[:3] != "V%d " % channel:
            print("ttiQl355tp.get_voltage() format error", ret)
            return None
        return float(ret[3:])

Response: get set voltage from ttiQl355tp


Instruction:
Here you have a function in Python 3, explain what it does
def get_current_limit(self, channel):
        ret = self.ask("I%d?" % channel)
        if ret[:3] != "I%d " % channel:
            print("ttiQl355tp.get_current_limit() format error", ret)
            return None
        return float(ret[3:])

Response: get current limit of the current system


Instruction:
Here you have a function in Python 3, explain what it does
def set_voltage(self, value, channel=1):
        cmd = "V%d %f" % (channel, value)
        self.write(cmd)

Response: set voltage of the current channel


Instruction:
Write a Python 3 function that can
channel 1=OP1, 2=OP2, AUX is not supported

Response:
def set_current_limit(self, value, channel=1):
        """ channel: 1=OP1, 2=OP2, AUX is not supported"""
        cmd = "I%d %f" % (channel, value)
        self.write(cmd)


Instruction:
Write a Python 3 script to
wait until the device is ready.

Response:
def wait_for_ready(self, timeout=None, times=None, delay=None, delay_between=None, abort=None):
        '''Determine the ready state of the device and wait until device is ready.

        Parameters
        ----------
        timeout : int, float
            The maximum amount of time to wait in seconds. Reaching the timeout will raise a RuntimeError.
        times : int
            Maximum number of times reading the ready state.
        delay : int, float
            The number of seconds to sleep before checks. Defaults to 0.
        delay_between : int, float
            The number of seconds to sleep between each check. Defaults to 0.
        abort : Threading.Event
            Breaking the loop from other threads.

        Returns
        -------
        True if state is ready, else False.
        '''
        if delay:
            try:
                sleep(delay)
            except IOError:  # negative values
                pass
        if timeout is not None:
            if timeout < 0:
                raise ValueError("timeout is smaller than 0")
            else:
                stop = time() + timeout
        times_checked = 0
        while not self.is_ready:
            now = time()
            times_checked += 1
            if abort and abort.is_set():
                False
            if timeout is not None and stop <= now:
                raise RuntimeError('Time out while waiting for ready in %s, module %s' % (self.name, self.__class__.__module__))
            if times and times > times_checked:
                False
            if delay_between:
                try:
                    sleep(delay_between)
                except IOError:  # negative values
                    pass
        return True


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _set_dac_value(self, channel, value):
        '''Write DAC
        '''
        self._intf.write(self._base_addr + self.MAX_5380_ADD, array('B', pack('B', value)))

Response: Write the value to the DAC.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _set_dac_value(self, channel, value):
        '''Write DAC
        '''
        # DAC value cannot be -128
        if value == -128:
            value = -127
        if value < 0:
            sign = 1
        else:
            sign = 0
        value = (sign << 7) | (0x7F & abs(value))
        self._intf.write(self._base_addr + self.DS_4424_ADD, array('B', pack('BB', channel, value)))

Response: Write the value to the specified channel of the DAC.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def read_eeprom_calibration(self, temperature=False):  # use default values for temperature, EEPROM values are usually not calibrated and random
        '''Reading EEPROM calibration for power regulators and temperature
        '''
        header = self.get_format()
        if header == self.HEADER_V2:
            data = self._read_eeprom(self.CAL_DATA_ADDR, size=calcsize(self.CAL_DATA_V2_FORMAT))
            for idx, channel in enumerate(self._ch_cal.iterkeys()):
                ch_data = data[idx * calcsize(self.CAL_DATA_CH_V2_FORMAT):(idx + 1) * calcsize(self.CAL_DATA_CH_V2_FORMAT)]
                values = unpack_from(self.CAL_DATA_CH_V2_FORMAT, ch_data)
                self._ch_cal[channel]['name'] = "".join([c for c in values[0] if (c in string.printable)])  # values[0].strip()
                self._ch_cal[channel]['default'] = values[1]
                self._ch_cal[channel]['ADCI']['gain'] = values[2]
                self._ch_cal[channel]['ADCI']['offset'] = values[3]
                self._ch_cal[channel]['ADCI']['iq_gain'] = values[4]
                self._ch_cal[channel]['ADCI']['iq_offset'] = values[5]
                self._ch_cal[channel]['ADCV']['gain'] = values[6]
                self._ch_cal[channel]['ADCV']['offset'] = values[7]
                self._ch_cal[channel]['DACV']['gain'] = values[8]
                self._ch_cal[channel]['DACV']['offset'] = values[9]
                if temperature:
                    self._ch_cal[channel]['NTC']['B_NTC'] = values[10]
                    self._ch_cal[channel]['NTC']['R1'] = values[11]
                    self._ch_cal[channel]['NTC']['R2'] = values[12]
                    self._ch_cal[channel]['NTC']['R4'] = values[13]
                    self._ch_cal[channel]['NTC']['R_NTC_25'] = values[14]
                    self._ch_cal[channel]['NTC']['VREF'] = values[15]
        else:
            raise ValueError('EEPROM data format not supported (header: %s)' % header)

Response: Reads the EEPROM calibration for power regulators and temperature.


Instruction:
Here you have a function in Python 3, explain what it does
def set_current_limit(self, channel, value, unit='A'):
        '''Setting current limit

        Note: same limit for all channels.
        '''
        dac_offset = self._ch_cal[channel]['DACI']['offset']
        dac_gain = self._ch_cal[channel]['DACI']['gain']

        if unit == 'raw':
            value = value
        elif unit == 'A':
            value = int((value - dac_offset) / dac_gain)
        elif unit == 'mA':
            value = int((value / 1000 - dac_offset) / dac_gain)
        else:
            raise TypeError("Invalid unit type.")

        DacMax5380._set_dac_value(self, channel, value)

Response: Set current limit for all channels.


Instruction:
Here you have a function in Python 3, explain what it does
def set_position(self, x, y, speed=None):
        ''' Move chuck to absolute position in um'''
        if speed:
            self._intf.write('MoveChuckSubsite %1.1f %1.1f R Y %d' % (x, y, speed))
        else:
            self._intf.write('MoveChuckSubsite %1.1f %1.1f R Y' % (x, y))

Response: Move chuck to absolute position in um


Instruction:
Make a summary of the following Python 3 code
def move_position(self, dx, dy, speed=None):
        ''' Move chuck relative to actual position in um'''
        if speed:
            self._intf.write('MoveChuckPosition %1.1f %1.1f R Y %d' % (dx, dy, speed))
        else:
            self._intf.write('MoveChuckPosition %1.1f %1.1f R Y' % (dx, dy))

Response: Move chuck relative to actual position in um


Instruction:
How would you code a function in Python 3 to
return the current read chuck position.

Response:
def get_position(self):
        ''' Read chuck position (x, y, z)'''
        reply = self._intf.query('ReadChuckPosition Y H')[2:]
        return [float(i) for i in reply.split()]


Instruction:
Can you implement a function in Python 3 that
moves chuck to wafer map chip index

Response:
def get_die(self):
        ''' Move chuck to wafer map chip index'''
        reply = self._intf.query('ReadMapPosition')
        values = reply[2:].split(' ')
        return (int(values[0]), int(values[1]))


Instruction:
Can you generate the documentation for the following Python 3 function
def clear(self):
        'Clear tracks in memory - all zero'
        for track in self._tracks:
            self._tracks[track].setall(False)

Response: Clear tracks in memory - all zero


Instruction:
Can you generate the documentation for the following Python 3 function
def get_data(self):
        ''' Reading data from SiTCP FIFO (via TCP).

        Returns
        -------
        array : numpy.ndarray
            Array of unsigned integers (32 bit).
        '''
        fifo_size = self._intf._get_tcp_data_size()
        fifo_int_size = int((fifo_size - (fifo_size % 4)) / 4)
        data = self._intf._get_tcp_data(fifo_int_size * 4)
        return np.frombuffer(data, dtype=np.dtype('<u4'))

Response: Reads the data from the TCP FIFO. Returns ------- array : numpy. ndarray


Instruction:
Implement a Python 3 function for
sending data to the NX - OS device via TCP.

Response:
def set_data(self, data):
        ''' Sending data to via TCP.

        Parameters
        ----------
        data : array
            Array of unsigned integers (32 bit).
        '''
        data = array.array('B', struct.unpack("{}B".format(len(data) * 4), struct.pack("{}I".format(len(data)), *data)))
        self._intf._send_tcp_data(data)


Instruction:
Explain what the following Python 3 code does
def program_global_reg(self):

        self._clear_strobes()

        gr_size = len(self['GLOBAL_REG'][:])  # get the size
        self['SEQ']['SHIFT_IN'][0:gr_size] = self['GLOBAL_REG'][:]  # this will be shifted out
        self['SEQ']['GLOBAL_SHIFT_EN'][0:gr_size] = bitarray(gr_size * '1')  # this is to enable clock
        self['SEQ']['GLOBAL_CTR_LD'][gr_size + 1:gr_size + 2] = bitarray("1")  # load signals
        self['SEQ']['GLOBAL_DAC_LD'][gr_size + 1:gr_size + 2] = bitarray("1")

        # Execute the program (write bits to output pins)
        # + 1 extra 0 bit so that everything ends on LOW instead of HIGH
        self._run_seq(gr_size + 3)

Response: This function sends the global register to the chip and then reads the values from the chip into the chip.


Instruction:
Can you generate the documentation for the following Python 3 function
def program_pixel_reg(self, enable_receiver=True):

        self._clear_strobes()

        # enable receiver it work only if pixel register is enabled/clocked
        self['PIXEL_RX'].set_en(enable_receiver)

        px_size = len(self['PIXEL_REG'][:])  # get the size
        self['SEQ']['SHIFT_IN'][0:px_size] = self['PIXEL_REG'][:]  # this will be shifted out
        self['SEQ']['PIXEL_SHIFT_EN'][0:px_size] = bitarray(px_size * '1')  # this is to enable clock

        self._run_seq(px_size + 1)

Response: Program the pixel register into the chip.


Instruction:
Can you create a Python 3 function that
runs the sequence generator.

Response:
def _run_seq(self, size):
        """
        Send the contents of self['SEQ'] to the chip and wait until it finishes.

        """

        # Write the sequence to the sequence generator (hw driver)
        self['SEQ'].write(size)  # write pattern to memory

        self['SEQ'].set_size(size)  # set size
        self['SEQ'].set_repeat(1)  # set repeat
        self['SEQ'].start()  # start

        while not self['SEQ'].get_done():
            time.sleep(0.01)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def write(self, addr, data):
        '''Write access.

        :param addr: i2c slave address
        :type addr: char
        :param data: array/list of bytes
        :type data: iterable
        :rtype: None

        '''
        self.set_addr(addr & 0xfe)
        self.set_data(data)
        self.set_size(len(data))
        self.start()
        while not self.is_ready:
            pass

Response: Write the contents of data to the specified i2c slave address.


Instruction:
Can you tell what is the following Python 3 function doing
def read(self, addr, size):
        '''Read access.

        :param addr: i2c slave address
        :type addr: char
        :param size: size of transfer
        :type size: int
        :returns: data byte array
        :rtype: array.array('B')

        '''
        self.set_addr(addr | 0x01)
        self.set_size(size)
        self.start()
        while not self.is_ready:
            pass

        return self.get_data(size)

Response: Read the next set of bytes from the slave.


Instruction:
Can you generate the documentation for the following Python 3 function
def full_parser():
    global _full_parser

    if _full_parser:
        return _full_parser

    luigi_parser = luigi.cmdline_parser.CmdlineParser.get_instance()
    if not luigi_parser:
        return None

    # build the full argument parser with luigi helpers
    root_task = luigi_parser.known_args.root_task
    _full_parser = luigi_parser._build_parser(root_task)

    logger.debug("build full luigi argument parser")

    return _full_parser

Response: Returns the full argument parser for the luigi command line parser.


Instruction:
Write a Python 3 script for
returning a new ArgumentParser instance that only contains paremeter actions of the root task.

Response:
def root_task_parser():
    """
    Returns a new *ArgumentParser* instance that only contains paremeter actions of the root task.
    The returned instance is cached.
    """
    global _root_task_parser

    if _root_task_parser:
        return _root_task_parser

    luigi_parser = luigi.cmdline_parser.CmdlineParser.get_instance()
    if not luigi_parser:
        return None

    root_task = luigi_parser.known_args.root_task

    # get all root task parameter destinations
    root_dests = []
    for task_name, _, param_name, _ in luigi.task_register.Register.get_all_params():
        if task_name == root_task:
            root_dests.append(param_name)

    # create a new parser and add all root actions
    _root_task_parser = ArgumentParser(add_help=False)
    for action in list(full_parser()._actions):
        if not action.option_strings or action.dest in root_dests:
            _root_task_parser._add_action(action)

    logger.debug("build luigi argument parser for root task {}".format(root_task))

    return _root_task_parser


Instruction:
Implement a function in Python 3 to
return the list of command line arguments that do not belong to the root task.

Response:
def global_cmdline_args():
    """
    Returns the list of command line arguments that do not belong to the root task. The returned
    list is cached. Example:

    .. code-block:: python

        global_cmdline_args()
        # -> ["--local-scheduler"]
    """
    global _global_cmdline_args

    if _global_cmdline_args:
        return _global_cmdline_args

    luigi_parser = luigi.cmdline_parser.CmdlineParser.get_instance()
    if not luigi_parser:
        return None

    _global_cmdline_args = root_task_parser().parse_known_args(luigi_parser.cmdline_args)[1]

    return _global_cmdline_args


Instruction:
Implement a Python 3 function for
returning a dictionary of global command line arguments to their current values.

Response:
def global_cmdline_values():
    """
    Returns a dictionary of global command line arguments (computed with
    :py:func:`global_cmdline_args`) to their current values. The returnd dictionary is cached.
    Example:

    .. code-block:: python

        global_cmdline_values()
        # -> {"core_local_scheduler": True}
    """
    global _global_cmdline_values

    if _global_cmdline_values:
        return _global_cmdline_values

    luigi_parser = luigi.cmdline_parser.CmdlineParser.get_instance()
    if not luigi_parser:
        return None

    # go through all actions of the full luigi parser and compare option strings
    # with the global cmdline args
    parser = full_parser()
    global_args = global_cmdline_args()
    _global_cmdline_values = {}
    for action in parser._actions:
        if any(arg in action.option_strings for arg in global_args):
            _global_cmdline_values[action.dest] = getattr(luigi_parser.known_args, action.dest)

    return _global_cmdline_values


Instruction:
Can you generate a brief explanation for the following Python 3 code
def add_cmdline_arg(args, arg, *values):
    if arg not in args:
        args = list(args) + [arg] + list(values)
    return args

Response: Adds a command line argument arg to a list of argument values.


Instruction:
Create a Python 3 function for
removing the command line argument arg from a list of arguments args.

Response:
def remove_cmdline_arg(args, arg, n=1):
    """
    Removes the command line argument *args* from a list of arguments *args*, e.g. as returned from
    :py:func:`global_cmdline_args`. When *n* is 1 or less, only the argument is removed. Otherwise,
    the following *n-1* values are removed. Example:

    .. code-block:: python

        args = global_cmdline_values()
        # -> ["--local-scheduler", "--workers", "4"]

        remove_cmdline_arg(args, "--local-scheduler")
        # -> ["--workers", "4"]

        remove_cmdline_arg(args, "--workers", 2)
        # -> ["--local-scheduler"]
    """
    if arg in args:
        idx = args.index(arg)
        args = list(args)
        del args[idx:idx + max(n, 1)]
    return args


Instruction:
Can you generate the documentation for the following Python 3 function
def split_transfer_kwargs(kwargs, skip=None):
    skip = make_list(skip) if skip else []
    transfer_kwargs = {
        name: kwargs.pop(name)
        for name in ["cache", "prefer_cache", "retries", "retry_delay"]
        if name in kwargs and name not in skip
    }
    return transfer_kwargs, kwargs

Response:
Takes keyword arguments kwargs and returns them in two separate dictionaries depending on their
    content.


Instruction:
Can you generate the documentation for the following Python 3 function
def job_data(cls, job_id=dummy_job_id, branches=None, **kwargs):
        return dict(job_id=job_id, branches=branches or [])

Response: Returns a dictionary containing default job submission information such as the job_id and branches.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def job_data(cls, job_id=dummy_job_id, status=None, code=None, error=None, **kwargs):
        return dict(job_id=job_id, status=status, code=code, error=error)

Response: Returns a dictionary containing default job status information such as the job_id status code and error message.


Instruction:
Write a Python 3 function that can
get the attribute of the actial task named name.

Response:
def _get_task_attribute(self, name, fallback=False):
        """
        Return an attribute of the actial task named ``<workflow_type>_<name>``.
        When the attribute does not exist and *fallback* is *True*, try to return the task attribute
        simply named *name*. In any case, if a requested task attribute is eventually not found, an
        AttributeError is raised.
        """
        attr = "{}_{}".format(self.workflow_type, name)
        if not fallback:
            return getattr(self.task, attr)
        else:
            value = getattr(self.task, attr, no_value)
            return value if value != no_value else getattr(self.task, name)


Instruction:
Can you generate the documentation for the following Python 3 function
def output(self):
        task = self.task

        # get the directory where the control outputs are stored
        out_dir = self._get_task_attribute("output_directory")()

        # define outputs
        outputs = OrderedDict()
        postfix = self._get_task_attribute("output_postfix")()

        # a file containing the submission data, i.e. job ids etc
        submission_file = "{}_submission{}.json".format(self.workflow_type, postfix)
        outputs["submission"] = out_dir.child(submission_file, type="f")
        outputs["submission"].optional = True

        # a file containing status data when the jobs are done
        if not task.no_poll:
            status_file = "{}_status{}.json".format(self.workflow_type, postfix)
            outputs["status"] = out_dir.child(status_file, type="f")
            outputs["status"].optional = True

        # update with upstream output when not just controlling running jobs
        if not task.is_controlling_remote_jobs():
            outputs.update(super(BaseRemoteWorkflowProxy, self).output())

        return outputs

Response: Returns the default workflow outputs in an ordered dictionary.


Instruction:
Can you generate the documentation for the following Python 3 function
def dump_submission_data(self):
        # renew the dashboard config
        self.submission_data["dashboard_config"] = self.dashboard.get_persistent_config()

        # write the submission data to the output file
        self._outputs["submission"].dump(self.submission_data, formatter="json", indent=4)

Response: Dumps the current submission data to the submission file.


Instruction:
Can you generate the documentation for the following Python 3 function
def run(self):
        task = self.task
        self._outputs = self.output()

        # create the job dashboard interface
        self.dashboard = task.create_job_dashboard() or NoJobDashboard()

        # read submission data and reset some values
        submitted = not task.ignore_submission and self._outputs["submission"].exists()
        if submitted:
            self.submission_data.update(self._outputs["submission"].load(formatter="json"))
            task.tasks_per_job = self.submission_data.tasks_per_job
            self.dashboard.apply_config(self.submission_data.dashboard_config)

        # when the branch outputs, i.e. the "collection" exists, just create dummy control outputs
        if "collection" in self._outputs and self._outputs["collection"].exists():
            self.touch_control_outputs()

        # cancel jobs?
        elif self._cancel_jobs:
            if submitted:
                self.cancel()

        # cleanup jobs?
        elif self._cleanup_jobs:
            if submitted:
                self.cleanup()

        # submit and/or wait while polling
        else:
            # maybe set a tracking url
            tracking_url = self.dashboard.create_tracking_url()
            if tracking_url:
                task.set_tracking_url(tracking_url)

            # ensure the output directory exists
            if not submitted:
                self._outputs["submission"].parent.touch()

            # at this point, when the status file exists, it is considered outdated
            if "status" in self._outputs:
                self._outputs["status"].remove()

            try:
                # instantiate the configured job file factory, not kwargs yet
                self.job_file_factory = self.create_job_file_factory()

                # submit
                if not submitted:
                    # set the initial list of unsubmitted jobs
                    branches = sorted(task.branch_map.keys())
                    branch_chunks = list(iter_chunks(branches, task.tasks_per_job))
                    self.submission_data.unsubmitted_jobs = OrderedDict(
                        (i + 1, branches) for i, branches in enumerate(branch_chunks)
                    )
                    self.submit()

                    # sleep once to give the job interface time to register the jobs
                    post_submit_delay = self._get_task_attribute("post_submit_delay")()
                    if post_submit_delay:
                        time.sleep(post_submit_delay)

                # start status polling when a) no_poll is not set, or b) the jobs were already
                # submitted so that failed jobs are resubmitted after a single polling iteration
                if not task.no_poll or submitted:
                    self.poll()

            finally:
                # in any event, cleanup the job file
                if self.job_file_factory:
                    self.job_file_factory.cleanup_dir(force=False)

Response: This method is called by the base class to start the processing of jobs and creates the job dashboard interface and the status polling and cleanup jobs.


Instruction:
Can you tell what is the following Python 3 function doing
def cancel(self):
        task = self.task

        # get job ids from submission data
        job_ids = [
            d["job_id"] for d in self.submission_data.jobs.values()
            if d["job_id"] not in (self.submission_data.dummy_job_id, None)
        ]
        if not job_ids:
            return

        # cancel jobs
        task.publish_message("going to cancel {} jobs".format(len(job_ids)))
        errors = self.job_manager.cancel_batch(job_ids)

        # print errors
        if errors:
            print("{} error(s) occured while cancelling {} job(s) of task {}:".format(
                len(errors), len(job_ids), task.task_id))
            tmpl = "    {}"
            for i, err in enumerate(errors):
                print(tmpl.format(err))
                if i + 1 >= self.show_errors:
                    remaining = len(errors) - self.show_errors
                    if remaining > 0:
                        print("    ... and {} more".format(remaining))
                    break

        # inform the dashboard
        for job_num, job_data in six.iteritems(self.submission_data.jobs):
            task.forward_dashboard_event(self.dashboard, job_data, "action.cancel", job_num)

Response:
Cancels running jobs. The job ids are read from the submission file which has to exist
        for obvious reasons.


Instruction:
Explain what the following Python 3 code does
def cleanup(self):
        task = self.task

        # get job ids from submission data
        job_ids = [
            d["job_id"] for d in self.submission_data.jobs.values()
            if d["job_id"] not in (self.submission_data.dummy_job_id, None)
        ]
        if not job_ids:
            return

        # cleanup jobs
        task.publish_message("going to cleanup {} jobs".format(len(job_ids)))
        errors = self.job_manager.cleanup_batch(job_ids)

        # print errors
        if errors:
            print("{} error(s) occured while cleaning up {} job(s) of task {}:".format(
                len(errors), len(job_ids), task.task_id))
            tmpl = "    {}"
            for i, err in enumerate(errors):
                print(tmpl.format(err))
                if i + 1 >= self.show_errors:
                    remaining = len(errors) - self.show_errors
                    if remaining > 0:
                        print("    ... and {} more".format(remaining))
                    break

Response: Clean up jobs on the remote run location.


Instruction:
Make a summary of the following Python 3 code
def submit(self, retry_jobs=None):
        task = self.task

        # helper to check if a job can be skipped
        # rule: skip a job when only_missing is set to True and all its branch tasks are complete
        def skip_job(job_num, branches):
            if not task.only_missing:
                return False
            elif job_num in self.skip_jobs:
                return self.skip_jobs[job_num]
            else:
                self.skip_jobs[job_num] = all(task.as_branch(b).complete() for b in branches)
                # when the job is skipped, write a dummy entry into submission data
                if self.skip_jobs[job_num]:
                    self.submission_data.jobs[job_num] = self.submission_data_cls.job_data(
                        branches=branches)
                return self.skip_jobs[job_num]

        # collect data of jobs that should be submitted: num -> branches
        submit_jobs = OrderedDict()

        # handle jobs for resubmission
        if retry_jobs:
            for job_num, branches in six.iteritems(retry_jobs):
                # even retry jobs can be skipped
                if skip_job(job_num, branches):
                    continue

                # the number of parallel jobs might be reached as well
                # in that case, add the jobs back to the unsubmitted ones and update the job id
                n = self.poll_data.n_active + len(submit_jobs)
                if n >= self.poll_data.n_parallel:
                    self.submission_data.unsubmitted_jobs[job_num] = branches
                    del self.submission_data.jobs[job_num]
                    continue

                # mark job for resubmission
                submit_jobs[job_num] = sorted(branches)

        # fill with jobs from the list of unsubmitted jobs until maximum number of parallel jobs is
        # reached
        new_jobs = OrderedDict()
        for job_num, branches in list(self.submission_data.unsubmitted_jobs.items()):
            # remove jobs that don't need to be submitted
            if skip_job(job_num, branches):
                del self.submission_data.unsubmitted_jobs[job_num]
                continue

            # do nothing when n_parallel is already reached
            n = self.poll_data.n_active + len(submit_jobs) + len(new_jobs)
            if n >= self.poll_data.n_parallel:
                continue

            # mark jobs for submission
            del self.submission_data.unsubmitted_jobs[job_num]
            new_jobs[job_num] = sorted(branches)

        # add new jobs to the jobs to submit, maybe also shuffle
        new_submission_data = OrderedDict()
        new_job_nums = list(new_jobs.keys())
        if task.shuffle_jobs:
            random.shuffle(new_job_nums)
        for job_num in new_job_nums:
            submit_jobs[job_num] = new_jobs[job_num]

        # when there is nothing to submit, dump the submission data to the output file and stop here
        if not submit_jobs:
            if retry_jobs or self.submission_data.unsubmitted_jobs:
                self.dump_submission_data()
            return new_submission_data

        # create job submission files
        job_files = [self.create_job_file(*tpl) for tpl in six.iteritems(submit_jobs)]

        # log some stats
        dst_info = self.destination_info() or ""
        dst_info = dst_info and (", " + dst_info)
        task.publish_message("going to submit {} {} job(s){}".format(
            len(submit_jobs), self.workflow_type, dst_info))

        # actual submission
        job_ids = self.submit_jobs(job_files)

        # store submission data
        errors = []
        for job_num, job_id in six.moves.zip(submit_jobs, job_ids):
            # handle errors
            error = (job_num, job_id) if isinstance(job_id, Exception) else None
            if error:
                errors.append((job_num, job_id))
                job_id = self.submission_data_cls.dummy_job_id

            # build the job data
            branches = submit_jobs[job_num]
            job_data = self.submission_data_cls.job_data(job_id=job_id, branches=branches)
            self.submission_data.jobs[job_num] = job_data
            new_submission_data[job_num] = job_data

            # set the attempt number in the submission data
            self.submission_data.attempts.setdefault(job_num, 0)

            # inform the dashboard
            task.forward_dashboard_event(self.dashboard, job_data, "action.submit", job_num)

        # dump the submission data to the output file
        self.dump_submission_data()

        # raise exceptions or log
        if errors:
            print("{} error(s) occured during job submission of task {}:".format(
                len(errors), task.task_id))
            tmpl = "    job {}: {}"
            for i, tpl in enumerate(errors):
                print(tmpl.format(*tpl))
                if i + 1 >= self.show_errors:
                    remaining = len(errors) - self.show_errors
                    if remaining > 0:
                        print("    ... and {} more".format(remaining))
                    break
        else:
            task.publish_message("submitted {} job(s)".format(len(submit_jobs)) + dst_info)

        return new_submission_data

Response: Submit all jobs in the current branch set.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def poll(self):
        task = self.task

        # total job count
        n_jobs = len(self.submission_data)

        # store the number of consecutive polling failures and get the maximum number of polls
        n_poll_fails = 0
        if task.walltime == NO_FLOAT:
            max_polls = sys.maxsize
        else:
            max_polls = int(math.ceil((task.walltime * 3600.) / (task.poll_interval * 60.)))

        # update variable attributes for polling
        self.poll_data.n_finished_min = task.acceptance * (1 if task.acceptance > 1 else n_jobs)
        self.poll_data.n_failed_max = task.tolerance * (1 if task.tolerance > 1 else n_jobs)

        # track finished and failed jobs in dicts holding status data
        finished_jobs = OrderedDict()
        failed_jobs = OrderedDict()

        # start the poll loop
        for i in six.moves.range(max_polls):
            # sleep after the first iteration
            if i > 0:
                time.sleep(task.poll_interval * 60)

            # determine the currently active jobs, i.e., the jobs whose states should be checked
            active_jobs = OrderedDict()
            for job_num, data in six.iteritems(self.submission_data.jobs):
                if job_num in finished_jobs or job_num in failed_jobs:
                    continue
                if self.skip_jobs.get(job_num):
                    finished_jobs[job_num] = self.status_data_cls.job_data(
                        status=self.job_manager.FINISHED, code=0)
                else:
                    data = data.copy()
                    if not data["job_id"]:
                        data["job_id"] = self.status_data_cls.dummy_job_id
                    active_jobs[job_num] = data
            self.poll_data.n_active = len(active_jobs)

            # query job states
            job_ids = [data["job_id"] for data in six.itervalues(active_jobs)]  # noqa: F812
            _states, errors = self.job_manager.query_batch(job_ids)
            if errors:
                print("{} error(s) occured during job status query of task {}:".format(
                    len(errors), task.task_id))
                tmpl = "    {}"
                for i, err in enumerate(errors):
                    print(tmpl.format(err))
                    if i + 1 >= self.show_errors:
                        remaining = len(errors) - self.show_errors
                        if remaining > 0:
                            print("    ... and {} more".format(remaining))
                        break

                n_poll_fails += 1
                if task.poll_fails > 0 and n_poll_fails > task.poll_fails:
                    raise Exception("poll_fails exceeded")
                else:
                    continue
            else:
                n_poll_fails = 0

            # states stores job_id's as keys, so replace them by using job_num's
            states = OrderedDict()
            for job_num, data in six.iteritems(active_jobs):
                states[job_num] = self.status_data_cls.job_data(**_states[data["job_id"]])

            # store jobs per status and take further actions depending on the status
            pending_jobs = OrderedDict()
            running_jobs = OrderedDict()
            newly_failed_jobs = OrderedDict()
            retry_jobs = OrderedDict()
            for job_num, data in six.iteritems(states):
                if data["status"] == self.job_manager.PENDING:
                    pending_jobs[job_num] = data
                    task.forward_dashboard_event(self.dashboard, data, "status.pending", job_num)

                elif data["status"] == self.job_manager.RUNNING:
                    running_jobs[job_num] = data
                    task.forward_dashboard_event(self.dashboard, data, "status.running", job_num)

                elif data["status"] == self.job_manager.FINISHED:
                    finished_jobs[job_num] = data
                    self.poll_data.n_active -= 1
                    task.forward_dashboard_event(self.dashboard, data, "status.finished", job_num)

                elif data["status"] in (self.job_manager.FAILED, self.job_manager.RETRY):
                    newly_failed_jobs[job_num] = data
                    self.poll_data.n_active -= 1

                    # retry or ultimately failed?
                    if self.job_retries[job_num] < task.retries:
                        self.job_retries[job_num] += 1
                        self.submission_data.attempts[job_num] += 1
                        data["status"] = self.job_manager.RETRY
                        retry_jobs[job_num] = self.submission_data.jobs[job_num]["branches"]
                        task.forward_dashboard_event(self.dashboard, data, "status.retry", job_num)
                    else:
                        failed_jobs[job_num] = data
                        task.forward_dashboard_event(self.dashboard, data, "status.failed", job_num)

                else:
                    raise Exception("unknown job status '{}'".format(data["status"]))

            # gather some counts
            n_pending = len(pending_jobs)
            n_running = len(running_jobs)
            n_finished = len(finished_jobs)
            n_retry = len(retry_jobs)
            n_failed = len(failed_jobs)
            n_unsubmitted = len(self.submission_data.unsubmitted_jobs)

            # log the status line
            counts = (n_pending, n_running, n_finished, n_retry, n_failed)
            if task.parallel_jobs > 0:
                counts = (n_unsubmitted,) + counts
            status_line = self.job_manager.status_line(counts, last_counts=True, sum_counts=n_jobs,
                color=True, align=task.align_status_line)
            task.publish_message(status_line)
            self.last_status_counts = counts

            # inform the scheduler about the progress
            task.publish_progress(100. * n_finished / n_jobs)

            # log newly failed jobs
            if newly_failed_jobs:
                print("{} failed job(s) in task {}:".format(len(newly_failed_jobs), task.task_id))
                tmpl = "    job: {}, branches: {}, id: {job_id}, status: {status}, code: {code}, " \
                    "error: {error}"
                for i, (job_num, data) in enumerate(six.iteritems(newly_failed_jobs)):
                    branches = self.submission_data.jobs[job_num]["branches"]
                    print(tmpl.format(job_num, ",".join(str(b) for b in branches), **data))
                    if i + 1 >= self.show_errors:
                        remaining = len(newly_failed_jobs) - self.show_errors
                        if remaining > 0:
                            print("    ... and {} more".format(remaining))
                        break

            # infer the overall status
            reached_end = n_jobs == n_finished + n_failed
            finished = n_finished >= self.poll_data.n_finished_min
            failed = n_failed > self.poll_data.n_failed_max
            unreachable = n_jobs - n_failed < self.poll_data.n_finished_min
            if finished:
                # write status output
                if "status" in self._outputs:
                    status_data = self.status_data_cls()
                    status_data.jobs.update(finished_jobs)
                    status_data.jobs.update(states)
                    self._outputs["status"].dump(status_data, formatter="json", indent=4)
                break
            elif failed:
                failed_nums = [job_num for job_num in failed_jobs if job_num not in retry_jobs]
                raise Exception("tolerance exceeded for jobs {}".format(failed_nums))
            elif unreachable:
                err = None
                if reached_end:
                    err = "acceptance of {} not reached, total jobs: {}, failed jobs: {}"
                elif task.check_unreachable_acceptance:
                    err = "acceptance of {} unreachable, total jobs: {}, failed jobs: {}"
                if err:
                    raise Exception(err.format(self.poll_data.n_finished_min, n_jobs, n_failed))

            # configurable poll callback
            task.poll_callback(self.poll_data)

            # trigger automatic resubmission and submission of unsubmitted jobs
            self.submit(retry_jobs)

            # break when no polling is desired
            # we can get to this point when there was already a submission and the no_poll
            # parameter was set so that only failed jobs are resubmitted once
            if task.no_poll:
                break
        else:
            # walltime exceeded
            raise Exception("walltime exceeded")

Response: This method is called by the poll method in the job manager to check the status of the jobs in the submission.


Instruction:
Write a Python 3 script for
creating and saves dummy submission and status files. This method is called in case the branch task outputs exist.

Response:
def touch_control_outputs(self):
        """
        Creates and saves dummy submission and status files. This method is called in case the
        collection of branch task outputs exists.
        """
        task = self.task

        # create the parent directory
        self._outputs["submission"].parent.touch()

        # get all branch indexes and chunk them by tasks_per_job
        branch_chunks = list(iter_chunks(task.branch_map.keys(), task.tasks_per_job))

        # submission output
        if not self._outputs["submission"].exists():
            submission_data = self.submission_data.copy()
            # set dummy submission data
            submission_data.jobs.clear()
            for i, branches in enumerate(branch_chunks):
                job_num = i + 1
                submission_data.jobs[job_num] = self.submission_data_cls.job_data(branches=branches)
            self._outputs["submission"].dump(submission_data, formatter="json", indent=4)

        # status output
        if "status" in self._outputs and not self._outputs["status"].exists():
            status_data = self.status_data_cls()
            # set dummy status data
            for i, branches in enumerate(branch_chunks):
                job_num = i + 1
                status_data.jobs[job_num] = self.status_data_cls.job_data(
                    status=self.job_manager.FINISHED, code=0)
            self._outputs["status"].dump(status_data, formatter="json", indent=4)


Instruction:
Given the following Python 3 function, write the documentation
def forward_dashboard_event(self, dashboard, job_data, event, job_num):
        # possible events:
        #   - action.submit
        #   - action.cancel
        #   - status.pending
        #   - status.running
        #   - status.finished
        #   - status.retry
        #   - status.failed
        # forward to dashboard in any event by default
        return dashboard.publish(job_data, event, job_num)

Response: Forward the event to the dashboard.


Instruction:
Explain what the following Python 3 code does
def localize(self, mode="r", perm=None, parent_perm=None, **kwargs):
        if mode not in ("r", "w"):
            raise Exception("unknown mode '{}', use r or w".format(mode))

        # get additional arguments
        skip_copy = kwargs.pop("skip_copy", False)
        is_tmp = kwargs.pop("is_tmp", mode == "w")

        if mode == "r":
            if is_tmp:
                # create a temporary target
                tmp = self.__class__(is_tmp=self.ext(n=1) or True)

                # always copy
                self.copy_to_local(tmp)

                # yield the copy
                try:
                    yield tmp
                finally:
                    tmp.remove()
            else:
                # simply yield
                yield self

        else:  # write mode
            if is_tmp:
                # create a temporary target
                tmp = self.__class__(is_tmp=self.ext(n=1) or True)

                # copy when existing
                if not skip_copy and self.exists():
                    self.copy_to_local(tmp)

                # yield the copy
                try:
                    yield tmp

                    # move back again
                    if tmp.exists():
                        tmp.move_to_local(self, dir_perm=parent_perm)
                        self.chmod(perm)
                    else:
                        logger.warning("cannot move non-existing localized file target {!r}".format(
                            self))
                finally:
                    tmp.remove()
            else:
                # create the parent dir
                self.parent.touch(perm=parent_perm)

                # simply yield
                yield self

                if self.exists():
                    self.chmod(perm)

Response: A generator that yields the local version of the current file.


Instruction:
How would you code a function in Python 3 to
set up the command line parser for the index subprogram and adds it to the given sub_parsers.

Response:
def setup_parser(sub_parsers):
    """
    Sets up the command line parser for the *index* subprogram and adds it to *sub_parsers*.
    """
    parser = sub_parsers.add_parser("index", prog="law index", description="Create or update the"
        " (human-readable) law task index file ({}). This is only required for the shell"
        " auto-completion.".format(Config.instance().get("core", "index_file")))

    parser.add_argument("--modules", "-m", nargs="+", help="additional modules to traverse")
    parser.add_argument("--no-externals", "-e", action="store_true", help="skip external tasks")
    parser.add_argument("--remove", "-r", action="store_true", help="remove the index file and"
        " exit")
    parser.add_argument("--location", "-l", action="store_true", help="print the location of the"
        " index file and exit")
    parser.add_argument("--verbose", "-v", action="store_true", help="verbose output")


Instruction:
Can you write a function in Python 3 where it
executes the index subprogram with parsed commandline args.

Response:
def execute(args):
    """
    Executes the *index* subprogram with parsed commandline *args*.
    """
    index_file = Config.instance().get_expanded("core", "index_file")

    # just print the file location?
    if args.location:
        print(index_file)
        return

    # just remove the index file?
    if args.remove:
        if os.path.exists(index_file):
            os.remove(index_file)
            print("removed index file {}".format(index_file))
        return

    # get modules to lookup
    lookup = [m.strip() for m in Config.instance().keys("modules")]
    if args.modules:
        lookup += args.modules

    print("loading tasks from {} module(s)".format(len(lookup)))

    # loop through modules, import everything to load tasks
    for modid in lookup:
        if not modid:
            continue

        if args.verbose:
            sys.stdout.write("loading module '{}'".format(modid))

        try:
            import_module(modid)
        except Exception as e:
            if not args.verbose:
                print("Error in module '{}': {}".format(colored(modid, "red"), str(e)))
            else:
                print("\n\nError in module '{}':".format(colored(modid, "red")))
                traceback.print_exc()
            continue

        if args.verbose:
            print(", {}".format(colored("done", style="bright")))

    # determine tasks to write into the index file
    seen_families = []
    task_classes = []
    lookup = [Task]
    while lookup:
        cls = lookup.pop(0)
        lookup.extend(cls.__subclasses__())

        # skip already seen task families
        if cls.task_family in seen_families:
            continue
        seen_families.append(cls.task_family)

        # skip when explicitly excluded
        if cls.exclude_index:
            continue

        # skip external tasks
        is_external_task = issubclass(cls, ExternalTask)
        if args.no_externals and is_external_task:
            continue

        # skip non-external tasks without run implementation
        run_is_callable = callable(getattr(cls, "run", None))
        run_is_abstract = getattr(cls.run, "__isabstractmethod__", False)
        if not is_external_task and (not run_is_callable or run_is_abstract):
            continue

        task_classes.append(cls)

    def get_task_params(cls):
        params = []
        for attr in dir(cls):
            member = getattr(cls, attr)
            if isinstance(member, luigi.Parameter):
                exclude = getattr(cls, "exclude_params_index", set())
                if not multi_match(attr, exclude, any):
                    params.append(attr.replace("_", "-"))
        return params

    def index_line(cls, params):
        # format: "module_id:task_family:param param ..."
        return "{}:{}:{}".format(cls.__module__, cls.task_family, " ".join(params))

    stats = OrderedDict()

    # write the index file
    if not os.path.exists(os.path.dirname(index_file)):
        os.makedirs(os.path.dirname(index_file))

    with open(index_file, "w") as f:
        for cls in task_classes:
            # get prams
            params = get_task_params(cls)

            # fill stats
            if cls.__module__ not in stats:
                stats[cls.__module__] = []
            stats[cls.__module__].append((cls.task_family, params))

            f.write(index_line(cls, params) + "\n")

    # print stats
    if args.verbose:
        for mod, data in six.iteritems(stats):
            print("\nmodule '{}', {} task(s):".format(colored(mod, style="bright"), len(data)))
            for task_family, _ in data:
                print("    - {}".format(colored(task_family, "green")))
        print("")

    print("written {} task(s) to index file '{}'".format(len(task_classes), index_file))


Instruction:
Implement a function in Python 3 to
return a list of luigi - internal configuration parameters.

Response:
def get_global_parameters(config_names=("core", "scheduler", "worker", "retcode")):
    """
    Returns a list of global, luigi-internal configuration parameters. Each list item is a 4-tuple
    containing the configuration class, the parameter instance, the parameter name, and the full
    parameter name in the cli. When *config_names* is set, it should be a list of configuration
    class names that are exclusively taken into account.
    """
    params = []
    for cls in luigi.task.Config.__subclasses__():
        if config_names and cls.__name__ not in config_names:
            continue

        for attr in dir(cls):
            param = getattr(cls, attr)
            if not isinstance(param, luigi.Parameter):
                continue

            full_name = attr.replace("_", "-")
            if getattr(cls, "use_cmdline_section", True):
                full_name = "{}-{}".format(cls.__name__.replace("_", "-"), full_name)

            params.append((cls, param, attr, full_name))

    return params


Instruction:
How would you code a function in Python 3 to
return a relative path to a framgment file or directory.

Response:
def rel_path(anchor, *paths):
    """
    Returns a path made of framgment *paths* relativ to an *anchor* path. When *anchor* is a file,
    its absolute directory is used instead.
    """
    anchor = os.path.abspath(os.path.expandvars(os.path.expanduser(anchor)))
    if os.path.exists(anchor) and os.path.isfile(anchor):
        anchor = os.path.dirname(anchor)
    return os.path.normpath(os.path.join(anchor, *paths))


Instruction:
Can you tell what is the following Python 3 function doing
def law_home_path(*paths):
    home = os.getenv("LAW_HOME", "$HOME/.law")
    home = os.path.expandvars(os.path.expanduser(home))
    return os.path.normpath(os.path.join(home, *paths))

Response: Returns the path to the law home directory that defaults to the user s home directory optionally with paths.


Instruction:
Here you have a function in Python 3, explain what it does
def print_err(*args, **kwargs):
    sys.stderr.write(" ".join(str(arg) for arg in args) + "\n")
    if kwargs.get("flush", False):
        sys.stderr.flush()

Response:
print_err(*args, flush=False)
    Same as *print*, but outputs to stderr. If *flush* is *True*, stderr is flushed after printing.


Instruction:
Can you tell what is the following Python 3 function doing
def abort(msg=None, exitcode=1):
    if msg is not None:
        if exitcode in (None, 0):
            print(msg)
        else:
            print_err(msg)
    sys.exit(exitcode)

Response: Aborts the process with an exitcode.


Instruction:
How would you explain what the following Python 3 function does
def colored(msg, color=None, background=None, style=None, force=False):
    try:
        if not force and not os.isatty(sys.stdout.fileno()):
            return msg
    except:
        return msg

    color = colors.get(color, colors["default"])
    background = backgrounds.get(background, backgrounds["default"])

    if not isinstance(style, (tuple, list, set)):
        style = (style,)
    style = ";".join(str(styles.get(s, styles["default"])) for s in style)

    return "\033[{};{};{}m{}\033[0m".format(style, background, color, msg)

Response: Return a colored version of a string.


Instruction:
Explain what the following Python 3 code does
def multi_match(name, patterns, mode=any, regex=False):
    if not regex:
        return mode(fnmatch.fnmatch(name, pattern) for pattern in patterns)
    else:
        return mode(re.match(pattern, name) for pattern in patterns)

Response: Returns True if name matches multiple patterns.


Instruction:
Create a Python 3 function for
returning whether the given object is iterable lazily.

Response:
def is_lazy_iterable(obj):
    """
    Returns whether *obj* is iterable lazily, such as generators, range objects, etc.
    """
    return isinstance(obj,
        (types.GeneratorType, collections.MappingView, six.moves.range, enumerate))


Instruction:
Write a Python 3 function for
converting an object to a list and returns it.

Response:
def make_list(obj, cast=True):
    """
    Converts an object *obj* to a list and returns it. Objects of types *tuple* and *set* are
    converted if *cast* is *True*. Otherwise, and for all other types, *obj* is put in a new list.
    """
    if isinstance(obj, list):
        return list(obj)
    elif is_lazy_iterable(obj):
        return list(obj)
    elif isinstance(obj, (tuple, set)) and cast:
        return list(obj)
    else:
        return [obj]


Instruction:
How would you explain what the following Python 3 function does
def make_tuple(obj, cast=True):
    if isinstance(obj, tuple):
        return tuple(obj)
    elif is_lazy_iterable(obj):
        return tuple(obj)
    elif isinstance(obj, (list, set)) and cast:
        return tuple(obj)
    else:
        return (obj,)

Response: Converts an object to a tuple and returns it.


Instruction:
Can you write a function in Python 3 where it
flattens and returns a complex structured object *struct*.

Response:
def flatten(struct):
    """
    Flattens and returns a complex structured object *struct*.
    """
    if isinstance(struct, dict):
        return flatten(struct.values())
    elif isinstance(struct, (list, tuple, set)) or is_lazy_iterable(struct):
        objs = []
        for obj in struct:
            objs.extend(flatten(obj))
        return objs
    else:
        return [struct]


Instruction:
Can you generate the documentation for the following Python 3 function
def merge_dicts(*dicts, **kwargs):
    # get or infer the class
    cls = kwargs.get("cls", None)
    if cls is None:
        for d in dicts:
            if isinstance(d, dict):
                cls = d.__class__
                break
        else:
            raise TypeError("cannot infer cls as none of the passed objects is of type dict")

    # start merging
    merged_dict = cls()
    for d in dicts:
        if isinstance(d, dict):
            merged_dict.update(d)

    return merged_dict

Response: Takes multiple dicts and returns a single merged dict.


Instruction:
Given the following Python 3 function, write the documentation
def which(prog):
    executable = lambda path: os.path.isfile(path) and os.access(path, os.X_OK)

    # prog can also be a path
    dirname, _ = os.path.split(prog)
    if dirname:
        if executable(prog):
            return prog
    elif "PATH" in os.environ:
        for search_path in os.environ["PATH"].split(os.pathsep):
            path = os.path.join(search_path.strip('"'), prog)
            if executable(path):
                return path

    return None

Response: Pythonic which implementation. Returns the path to an executable by searching in the PATH environment variable.


Instruction:
Can you generate the documentation for the following Python 3 function
def map_struct(func, struct, cls=None, map_dict=True, map_list=True, map_tuple=False,
        map_set=False):
    """
    Applies a function *func* to each value of a complex structured object *struct* and returns the
    output in the same structure. Example:

    .. code-block:: python

       struct = {"foo": [123, 456], "bar": [{"1": 1}, {"2": 2}]}
       def times_two(i):
           return i * 2

       map_struct(struct, times_two)
       # -> {"foo": [246, 912], "bar": [{"1": 2}, {"2": 4}]}

    When *cls* is not *None*, it exclusively defines the class of objects that *func* is applied on.
    All other objects are unchanged. *map_dict*, *map_list*, *map_tuple* and *map_set* configure if
    objects of the respective types are traversed or mapped. The can be booleans or integer values
    that define the depth of that setting in the struct.
    """
    # interpret generators and views as lists
    if is_lazy_iterable(struct):
        struct = list(struct)

    valid_types = tuple()
    if map_dict:
        valid_types += (dict,)
        if isinstance(map_dict, int) and not isinstance(map_dict, bool):
            map_dict -= 1
    if map_list:
        valid_types += (list,)
        if isinstance(map_list, int) and not isinstance(map_list, bool):
            map_list -= 1
    if map_tuple:
        valid_types += (tuple,)
        if isinstance(map_tuple, int) and not isinstance(map_tuple, bool):
            map_tuple -= 1
    if map_set:
        valid_types += (set,)
        if isinstance(map_set, int) and not isinstance(map_set, bool):
            map_set -= 1

    # is an instance of cls?
    if cls is not None and isinstance(struct, cls):
        return func(struct)

    # traverse?
    elif isinstance(struct, valid_types):
        # create a new struct, treat tuples as lists for itertative item appending
        new_struct = struct.__class__() if not isinstance(struct, tuple) else []

        # create type-dependent generator and addition callback
        if isinstance(struct, (list, tuple)):
            gen = enumerate(struct)
            add = lambda _, value: new_struct.append(value)
        elif isinstance(struct, set):
            gen = enumerate(struct)
            add = lambda _, value: new_struct.add(value)
        else:  # dict
            gen = six.iteritems(struct)
            add = lambda key, value: new_struct.__setitem__(key, value)

        # recursively fill the new struct
        for key, value in gen:
            value = map_struct(func, value, cls=cls, map_dict=map_dict, map_list=map_list,
                map_tuple=map_tuple, map_set=map_set)
            add(key, value)

        # convert tuples
        if isinstance(struct, tuple):
            new_struct = struct.__class__(new_struct)

        return new_struct

    # when cls is set, just return
    elif cls is not None:
        return struct

    # apply func
    else:
        return func(struct)

Response: Applies a function func to each value of a complex structured object struct and returns the resulting structure.


Instruction:
Write a Python 3 script for
masking a complex structured object struct with a mask and returns the remaining values.

Response:
def mask_struct(mask, struct, replace=no_value):
    """
    Masks a complex structured object *struct* with a *mask* and returns the remaining values. When
    *replace* is set, masked values are replaced with that value instead of being removed. The
    *mask* can have a complex structure as well. Examples:

    .. code-block:: python

        struct = {"a": [1, 2], "b": [3, ["foo", "bar"]]}

        # simple example
        mask_struct({"a": [False, True], "b": False}, struct)
        # => {"a": [2]}

        # omitting mask information results in keeping values
        mask_struct({"a": [False, True]}, struct)
        # => {"a": [2], "b": [3, ["foo", "bar"]]}
    """
    # interpret lazy iterables lists
    if is_lazy_iterable(struct):
        struct = list(struct)

    # when mask is a bool, or struct is not a dict or sequence, apply the mask immediately
    if isinstance(mask, bool) or not isinstance(struct, (list, tuple, dict)):
        return struct if mask else replace

    # check list and tuple types
    elif isinstance(struct, (list, tuple)) and isinstance(mask, (list, tuple)):
        new_struct = []
        for i, val in enumerate(struct):
            if i >= len(mask):
                new_struct.append(val)
            else:
                repl = replace
                if isinstance(replace, (list, tuple)) and len(replace) > i:
                    repl = replace[i]
                val = mask_struct(mask[i], val, replace=repl)
                if val != no_value:
                    new_struct.append(val)

        return struct.__class__(new_struct) if new_struct else replace

    # check dict types
    elif isinstance(struct, dict) and isinstance(mask, dict):
        new_struct = struct.__class__()
        for key, val in six.iteritems(struct):
            if key not in mask:
                new_struct[key] = val
            else:
                repl = replace
                if isinstance(replace, dict) and key in replace:
                    repl = replace[key]
                val = mask_struct(mask[key], val, replace=repl)
                if val != no_value:
                    new_struct[key] = val
        return new_struct or replace

    # when this point is reached, mask and struct have incompatible types
    raise TypeError("mask and struct must have the same type, got '{}' and '{}'".format(type(mask),
            type(struct)))


Instruction:
Can you generate the documentation for the following Python 3 function
def readable_popen(*args, **kwargs):
    # force pipes
    kwargs["stdout"] = subprocess.PIPE
    kwargs["stderr"] = subprocess.STDOUT

    p = subprocess.Popen(*args, **kwargs)

    for line in iter(lambda: p.stdout.readline(), ""):
        if six.PY3:
            line = line.decode("utf-8")
        yield line.rstrip()

    # yield the process itself in the end
    p.communicate()
    yield p

Response: A generator that yields the output of a single process.


Instruction:
Write a Python 3 function for
creating a hash based on a given algorithm.

Response:
def create_hash(inp, l=10, algo="sha256"):
    """
    Takes an input *inp* and creates a hash based on an algorithm *algo*. For valid algorithms, see
    python's hashlib. *l* corresponds to the maximum length of the returned hash. Internally, the
    string representation of *inp* is used.
    """
    return getattr(hashlib, algo)(six.b(str(inp))).hexdigest()[:l]


Instruction:
How would you explain what the following Python 3 function does
def copy_no_perm(src, dst):
    shutil.copy(src, dst)
    perm = os.stat(dst).st_mode
    shutil.copystat(src, dst)
    os.chmod(dst, perm)

Response: Copy a file from src to dst except for permission bits.


Instruction:
Create a Python 3 function for
returning True if a file located at path is owned by the user with uid.

Response:
def user_owns_file(path, uid=None):
    """
    Returns whether a file located at *path* is owned by the user with *uid*. When *uid* is *None*,
    the user id of the current process is used.
    """
    if uid is None:
        uid = os.getuid()
    path = os.path.expandvars(os.path.expanduser(path))
    return os.stat(path).st_uid == uid


Instruction:
Here you have a function in Python 3, explain what it does
def human_bytes(n, unit=None):
    if n == 0:
        idx = 0
    elif unit:
        idx = byte_units.index(unit)
    else:
        idx = int(math.floor(math.log(abs(n), 1024)))
        idx = min(idx, len(byte_units))
    return n / 1024. ** idx, byte_units[idx]

Response:
Takes a number of bytes n assigns the best matching unit and returns the respective number
    and unit string in a tuple.


Instruction:
Can you generate the documentation for the following Python 3 function
def human_time_diff(*args, **kwargs):
    secs = float(datetime.timedelta(*args, **kwargs).total_seconds())
    parts = []
    for unit, mul in time_units:
        if secs / mul >= 1 or mul == 1:
            if mul > 1:
                n = int(math.floor(secs / mul))
                secs -= n * mul
            else:
                n = round(secs, 1)
            parts.append("{} {}{}".format(n, unit, "" if n == 1 else "s"))
    return ", ".join(parts)

Response: Returns a human readable time difference.


Instruction:
Write a Python 3 script to
return whether the exception e was raised due to an already existing file or directory.

Response:
def is_file_exists_error(e):
    """
    Returns whether the exception *e* was raised due to an already existing file or directory.
    """
    if six.PY3:
        return isinstance(e, FileExistsError)  # noqa: F821
    else:
        return isinstance(e, OSError) and e.errno == 17


Instruction:
Write a Python 3 function that can
send an email from sender to recipient with subject and content.

Response:
def send_mail(recipient, sender, subject="", content="", smtp_host="127.0.0.1", smtp_port=25):
    """
    Lightweight mail functionality. Sends an mail from *sender* to *recipient* with *subject* and
    *content*. *smtp_host* and *smtp_port* are forwarded to the ``smtplib.SMTP`` constructor. *True*
    is returned on success, *False* otherwise.
    """
    try:
        server = smtplib.SMTP(smtp_host, smtp_port)
    except Exception as e:
        logger = logging.getLogger(__name__)
        logger.warning("cannot create SMTP server: {}".format(e))
        return False

    header = "From: {}\r\nTo: {}\r\nSubject: {}\r\n\r\n".format(sender, recipient, subject)
    server.sendmail(sender, recipient, header + content)

    return True


Instruction:
How would you implement a function in Python 3 that
flushes all registered consumer streams.

Response:
def _flush(self):
        """
        Flushes all registered consumer streams.
        """
        for consumer in self.consumers:
            if not getattr(consumer, "closed", False):
                consumer.flush()


Instruction:
Can you tell what is the following Python 3 function doing
def _write(self, *args, **kwargs):
        for consumer in self.consumers:
            consumer.write(*args, **kwargs)

Response: Write to all registered consumer streams passing args and kwargs.


Instruction:
Implement a function in Python 3 to
write to the consumer stream when filter_fn evaluates to True

Response:
def _write(self, *args, **kwargs):
        """
        Writes to the consumer stream when *filter_fn* evaluates to *True*, passing *args* and
        *kwargs*.
        """
        if self.filter_fn(*args, **kwargs):
            self.stream.write(*args, **kwargs)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def get_default(self, section, option, default=None, type=None, expandvars=False,
            expanduser=False):
        """
        Returns the config value defined by *section* and *option*. When either the section or the
        option does not exist, the *default* value is returned instead. When *type* is set, it must
        be either `"str"`, `"int"`, `"float"`, or `"boolean"`. When *expandvars* is *True*,
        environment variables are expanded. When *expanduser* is *True*, user variables are
        expanded as well.
        """
        if self.has_section(section) and self.has_option(section, option):
            value = self.get(section, option)
            if isinstance(value, six.string_types):
                if expandvars:
                    value = os.path.expandvars(value)
                if expanduser:
                    value = os.path.expanduser(value)
            return value if not type else self._get_type_converter(type)(value)
        else:
            return default

Response: Returns the default value for the option in section.


Instruction:
Can you tell what is the following Python 3 function doing
def get_expanded(self, *args, **kwargs):
        kwargs.setdefault("expandvars", True)
        kwargs.setdefault("expanduser", True)
        return self.get_default(*args, **kwargs)

Response: Same as get_default but expandvars and expanduser arguments are set to True by default.


Instruction:
Write a Python 3 script to
update the current configuration with new data given as a dictionary.

Response:
def update(self, data, overwrite=None, overwrite_sections=True, overwrite_options=True):
        """
        Updates the currently stored configuration with new *data*, given as a dictionary. When
        *overwrite_sections* is *False*, sections in *data* that are already present in the current
        config are skipped. When *overwrite_options* is *False*, existing options are not
        overwritten. When *overwrite* is not *None*, both *overwrite_sections* and
        *overwrite_options* are set to its value.
        """
        if overwrite is not None:
            overwrite_sections = overwrite
            overwrite_options = overwrite

        for section, _data in six.iteritems(data):
            if not self.has_section(section):
                self.add_section(section)
            elif not overwrite_sections:
                continue

            for option, value in six.iteritems(_data):
                if overwrite_options or not self.has_option(section, option):
                    self.set(section, option, str(value))


Instruction:
Implement a Python 3 function for
updating the current configc with the config found in filename. All args and kwargs are passed to the update method.

Response:
def include(self, filename, *args, **kwargs):
        """
        Updates the current configc with the config found in *filename*. All *args* and *kwargs* are
        forwarded to :py:meth:`update`.
        """
        p = self.__class__(filename, skip_defaults=True, skip_fallbacks=True)
        self.update(p._sections, *args, **kwargs)


Instruction:
Can you write a function in Python 3 where it
returns all keys of a section in a list.

Response:
def keys(self, section, prefix=None):
        """
        Returns all keys of a *section* in a list. When *prefix* is set, only keys starting with
        that prefix are returned
        """
        return [key for key, _ in self.items(section) if (not prefix or key.startswith(prefix))]


Instruction:
Given the following Python 3 function, write the documentation
def sync_luigi_config(self, push=True, pull=True, expand=True):
        prefix = "luigi_"
        lparser = luigi.configuration.LuigiConfigParser.instance()

        if push:
            for section in self.sections():
                if not section.startswith(prefix):
                    continue
                lsection = section[len(prefix):]

                if not lparser.has_section(lsection):
                    lparser.add_section(lsection)

                for option in self.options(section):
                    if not lparser.has_option(lsection, option):
                        if expand:
                            value = self.get_expanded(section, option)
                        else:
                            value = self.get(section, option)
                        lparser.set(lsection, option, value)

        if pull:
            for lsection in lparser.sections():
                section = prefix + lsection

                if not self.has_section(section):
                    self.add_section(section)

                for option, value in lparser.items(lsection):
                    self.set(section, option, value)

Response: Synchronizes the configuration with the luigi configuration parser.


Instruction:
Given the following Python 3 function, write the documentation
def notify_telegram(title, content, token=None, chat=None, mention_user=None, **kwargs):
    # test import
    import telegram  # noqa: F401

    cfg = Config.instance()

    # get default token and chat
    if not token:
        token = cfg.get_expanded("notifications", "telegram_token")
    if not chat:
        chat = cfg.get_expanded("notifications", "telegram_chat")

    if not token or not chat:
        logger.warning("cannot send Telegram notification, token ({}) or chat ({}) empty".format(
            token, chat))
        return False

    # append the user to mention to the title
    # unless explicitly set to empty string
    mention_text = ""
    if mention_user is None:
        mention_user = cfg.get_expanded("notifications", "telegram_mention_user")
    if mention_user:
        mention_text = " (@{})".format(mention_user)

    # request data for the API call
    request = {
        "parse_mode": "Markdown",
    }

    # standard or attachment content?
    if isinstance(content, six.string_types):
        request["text"] = "{}{}\n\n{}".format(title, mention_text, content)
    else:
        # content is a dict, add some formatting
        request["text"] = "{}{}\n\n".format(title, mention_text)

        for key, value in content.items():
            request["text"] += "_{}_: {}\n".format(key, value)

    # extend by arbitrary kwargs
    request.update(kwargs)

    # threaded, non-blocking API communication
    thread = threading.Thread(target=_notify_telegram, args=(token, chat, request))
    thread.start()

    return True

Response: Send a Telegram notification and returns True on success.


Instruction:
Explain what the following Python 3 code does
def complete(self):
        if self.task.local_workflow_require_branches:
            return self._has_run
        else:
            return super(LocalWorkflowProxy, self).complete()

Response: Returns True if the task has completed successfully.


Instruction:
Here you have a function in Python 3, explain what it does
def run(self):
        if not self._has_yielded and not self.task.local_workflow_require_branches:
            self._has_yielded = True

            yield list(self.task.get_branch_tasks().values())

        self._has_run = True

Response: A generator that yields all branch - level task - related information.


Instruction:
Here you have a function in Python 3, explain what it does
def factory(**default_opts):
    def wrapper(decorator):
        @functools.wraps(decorator)
        def wrapper(fn=None, **opts):
            _opts = default_opts.copy()
            _opts.update(opts)

            def wrapper(fn):
                @functools.wraps(fn)
                def wrapper(*args, **kwargs):
                    return decorator(fn, _opts, *args, **kwargs)
                return wrapper

            return wrapper if fn is None else wrapper(fn)
        return wrapper
    return wrapper

Response: A function factory for tasks run methods.


Instruction:
Explain what the following Python 3 code does
def log(fn, opts, task, *args, **kwargs):
    _task = get_task(task)
    log = get_param(_task.log_file, _task.default_log_file)

    if log == "-" or not log:
        return fn(task, *args, **kwargs)
    else:
        # use the local target functionality to create the parent directory
        LocalFileTarget(log).parent.touch()
        with open_compat(log, "a", 1) as f:
            tee = TeeStream(f, sys.__stdout__)
            sys.stdout = tee
            sys.stderr = tee
            try:
                ret = fn(task, *args, **kwargs)
            except:
                traceback.print_exc(file=tee)
                raise
            finally:
                sys.stdout = sys.__stdout__
                sys.stderr = sys.__stderr__
                tee.flush()
        return ret

Response:
A decorator that wraps a bound method of a task and redirects output of both stdout and stderr to the file
            defined by the tasks s log_file parameter or the default_log_file attribute.


Instruction:
Can you create a Python 3 function that
wraps a bound method of a task and logs its execution time in a human readable format. Logs in info mode. When *publish_message* is *True*, the duration is also published as a task message to the scheduler.

Response:
def timeit(fn, opts, task, *args, **kwargs):
    """
    Wraps a bound method of a task and logs its execution time in a human readable format. Logs in
    info mode. When *publish_message* is *True*, the duration is also published as a task message to
    the scheduler.
    """
    start_time = time.time()
    try:
        return fn(task, *args, **kwargs)
    finally:
        duration = human_time_diff(seconds=round(time.time() - start_time, 1))

        # log
        timeit_logger = logger.getChild("timeit")
        timeit_logger.info("runtime of {}: {}".format(task.task_id, duration))

        # optionally publish a task message to the scheduler
        if opts["publish_message"] and callable(getattr(task, "publish_message", None)):
            task.publish_message("runtime: {}".format(duration))


Instruction:
Can you generate a brief explanation for the following Python 3 code
def get_voms_proxy_user():
    out = _voms_proxy_info(["--identity"])[1].strip()
    try:
        return re.match(r".*\/CN\=([^\/]+).*", out.strip()).group(1)
    except:
        raise Exception("no valid identity found in voms proxy: {}".format(out))

Response: Returns the owner of the voms proxy.


Instruction:
How would you implement a function in Python 3 that
checks if a voms proxy exists and returns True otherwise.

Response:
def check_voms_proxy_validity(log=False):
    """
    Returns *True* when a valid voms proxy exists, *False* otherwise. When *log* is *True*, a
    warning will be logged.
    """
    valid = _voms_proxy_info(["--exists"], silent=True)[0] == 0
    if log and not valid:
        logger.warning("no valid voms proxy found")
    return valid


Instruction:
Can you write a function in Python 3 where it
renews the voms proxy using a password and virtual organization name vo and a default lifetime of 8 days.

Response:
def renew_voms_proxy(passwd="", vo=None, lifetime="196:00"):
    """
    Renews the voms proxy using a password *passwd*, an optional virtual organization name *vo*, and
    a default *lifetime* of 8 days. The password is written to a temporary file first and piped into
    the renewal commad to ensure it is not visible in the process list.
    """
    with tmp_file() as (_, tmp):
        with open(tmp, "w") as f:
            f.write(passwd)

        cmd = "cat '{}' | voms-proxy-init --valid '{}'".format(tmp, lifetime)
        if vo:
            cmd += " -voms '{}'".format(vo)
        code, out, _ = interruptable_popen(cmd, shell=True, executable="/bin/bash",
            stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
        if code != 0:
            raise Exception("proxy renewal failed: {}".format(out))


Instruction:
Make a summary of the following Python 3 code
def delegate_voms_proxy_glite(endpoint, stdout=None, stderr=None, cache=True):
    # get the proxy file
    proxy_file = get_voms_proxy_file()
    if not os.path.exists(proxy_file):
        raise Exception("proxy file '{}' does not exist".format(proxy_file))

    if cache:
        if isinstance(cache, six.string_types):
            cache_file = cache
        else:
            cache_file = proxy_file + "_delegation_cache.json"

        def remove_cache():
            try:
                if os.path.exists(cache_file):
                    os.remove(cache_file)
            except OSError:
                pass

        # create the hash of the proxy file content
        with open(proxy_file, "r") as f:
            proxy_hash = create_hash(f.read())

        # already delegated?
        cache_data = {}
        if os.path.exists(cache_file):
            with open(cache_file, "r") as f:
                try:
                    cache_data = json.load(f)
                except:
                    remove_cache()

        # is the hash up-to-date?
        if cache_data.get("hash") != proxy_hash:
            remove_cache()
            cache_data = {}

        # proxy already delegated to that endpoint?
        elif endpoint in cache_data.get("ids", []):
            return str(cache_data["ids"][endpoint])

    # do the actual delegation
    delegation_id = uuid.uuid4().hex
    cmd = ["glite-ce-delegate-proxy", "-e", endpoint, delegation_id]
    code = interruptable_popen(cmd, stdout=stdout, stderr=stderr)[0]
    if code != 0:
        raise Exception("glite proxy delegation to endpoint {} failed".format(endpoint))

    if cache:
        # write the id back to the delegation file
        cache_data["hash"] = proxy_hash
        cache_data.setdefault("ids", {})[endpoint] = delegation_id
        with open(cache_file, "w") as f:
            json.dump(cache_data, f, indent=4)
        os.chmod(cache_file, 0o0600)

    return delegation_id

Response: Delegates the voms proxy to an endpoint using glite - ce - delegate - proxy.


Instruction:
Write a Python 3 function that can
return the formatter class whose name attribute is name.

Response:
def get_formatter(name, silent=False):
    """
    Returns the formatter class whose name attribute is *name*. When no class could be found and
    *silent* is *True*, *None* is returned. Otherwise, an exception is raised.
    """
    formatter = FormatterRegister.formatters.get(name)
    if formatter or silent:
        return formatter
    else:
        raise Exception("cannot find formatter '{}'".format(name))


Instruction:
Implement a function in Python 3 to
find the list of formatters that would accept the given file.

Response:
def find_formatters(path, silent=True):
    """
    Returns a list of formatter classes which would accept the file given by *path*. When no classes
    could be found and *silent* is *True*, an empty list is returned. Otherwise, an exception is
    raised.
    """
    formatters = [f for f in six.itervalues(FormatterRegister.formatters) if f.accepts(path)]
    if formatters or silent:
        return formatters
    else:
        raise Exception("cannot find formatter for path '{}'".format(path))


Instruction:
Explain what the following Python 3 code does
def find_formatter(name, path):
    if name == AUTO_FORMATTER:
        return find_formatters(path, silent=False)[0]
    else:
        return get_formatter(name, silent=False)

Response: Returns the formatter class that accepts path.


Instruction:
Can you implement a function in Python 3 that
encodes a list into a base64 encoded string via base64 encoding.

Response:
def encode_list(cls, value):
        """
        Encodes a list *value* into a string via base64 encoding.
        """
        encoded = base64.b64encode(six.b(" ".join(str(v) for v in value) or "-"))
        return encoded.decode("utf-8") if six.PY3 else encoded


Instruction:
Make a summary of the following Python 3 code
def get_args(self):
        return [
            self.task_cls.__module__,
            self.task_cls.__name__,
            self.encode_list(self.task_params),
            self.encode_list(self.branches),
            self.encode_bool(self.auto_retry),
            self.encode_list(self.dashboard_data),
        ]

Response: Returns the list of encoded job arguments.


Instruction:
Given the following Python 3 function, write the documentation
def load(*packages):
    for pkg in flatten(packages):
        if pkg in loaded_packages:
            logger.debug("skip contrib package '{}', already loaded".format(pkg))
            continue
        loaded_packages.append(pkg)

        mod = __import__("law.contrib.{}".format(pkg), globals(), locals(), [pkg])
        logger.debug("loaded contrib package '{}'".format(pkg))

        for attr in mod.__all__:
            if hasattr(law, attr):
                logger.info("cannot register 'law.contrib.{0}.{1}' to 'law.{1}', "
                    "already exists".format(pkg, attr))
            else:
                setattr(law, attr, getattr(mod, attr))
                law.__all__.append(attr)
                logger.debug("registered 'law.contrib.{0}.{1}' to 'law.{1}'".format(pkg, attr))

Response: Loads a list of contrib packages and adds members exposed in law. all.